#!/usr/bin/perl

use DBI;
use Cwd;
use POSIX;
use strict;

# these hashes map column names in tables to their order
my %observation;
my %calibrators;
($observation{'source'},$observation{'ra'},$observation{'dec'},$observation{'n_if'},
 $observation{'if_freq'},$observation{'if_nchans'},$observation{'if_bw'},$observation{'time'},
 $observation{'arrayname'},$observation{'project_code'},$observation{'rpfits_file'},
 $observation{'cycle_time'},$observation{'offsource'},$observation{'onsource'},
 $observation{'scantype'},$observation{'calcode'},$observation{'stations'},
 $observation{'header_number'},$observation{'num_processed'},$observation{'processed_ids'},
 $observation{'id'},$observation{'az'},$observation{'el'},$observation{'attenmm'},
 $observation{'subrefl'},$observation{'corr_cfg'},$observation{'airtemp'},
 $observation{'chi'},$observation{'evector'},$observation{'jyperk'},
 $observation{'pntra'},$observation{'pntdec'},$observation{'relhumid'},
 $observation{'smonrms'},$observation{'wind'},$observation{'winddir'},
 $observation{'xtsys'},$observation{'ytsys'},$observation{'xyphase'},$observation{'xcaljy'},
 $observation{'ycaljy'},$observation{'xgtp'},$observation{'ygtp'},$observation{'xsdo'},
 $observation{'ysdo'},$observation{'pointcor'},$observation{'point_time'},
 $observation{'point_source'},$observation{'point_az'},$observation{'point_el'},
 $observation{'usable'})=
    (0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,
     31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50);
($calibrators{'name'},$calibrators{'ra'},$calibrators{'dec'},$calibrators{'type'},
 $calibrators{'low_freq'},$calibrators{'high_freq'},$calibrators{'bandpass_21cm'},
 $calibrators{'bandpass_13cm'},$calibrators{'bandpass_6cm'},$calibrators{'bandpass_3cm'},
 $calibrators{'bandpass_12mm'},$calibrators{'bandpass_7mm'},$calibrators{'bandpass_3mm'})=
    (51,52,53,54,55,56,57,58,59,60,61,62,63);

# the query we use to get data to reduce - this gets all observations where the name matches
# that of a known calibrator, that hasn't yet been processed, and that has an IF with 2 GHz
# bandwidth, and returns up to 100 such observations, starting with the earliest and continuing
# in time order
my $get_query_parameters="*";
my $get_query_source="observation,calibrators";
my @get_query_constraints=("calibrators.name=observation.source","observation.num_processed=0",
			   "2048=any(observation.if_bw)","observation.time>'2009-06-01 00:00:00'",
			   "observation.usable=true");
my $get_query_order="time";
my $get_query_limit=100;

# some other defaults
# the working directory
my $working_directory="/data/MENTOK_2/ste616/flux_monitoring/staging_area";
my $minimum_ants=4;
my $minimum_time=10; # in seconds
my $bad_ants=3; # if the number of antennae on-source is less than or equal to this, we will
                # mark this observation as unusable
my @highlow_21cm=(0,2000);
my @highlow_13cm=(2000,4000);
my @highlow_6cm=(4000,6500);
my @highlow_3cm=(6500,10000);
my @highlow_12mm=(15000,25000);
my @highlow_7mm=(25000,55000);
my @highlow_3mm=(80000,110000);

my $output_debug=0;
my $delete_process=1;
my $bandpass_only=0;
my $redo_bandpass=0;
my $phase_only=0;
my $cm_obs_only=0;
my $mm_obs_only;
my $flux_only=0;
my $stop_after=0;
# get our our command line arguments
for (my $i=0;$i<=$#ARGV;$i++){
    if ($ARGV[$i] eq "-D"){
	# level up debug printing
	$output_debug++;
    } elsif ($ARGV[$i] eq "-F"){
	# only calibrate flux calibrators (which is the simplest thing to do)
	push @get_query_constraints,"calibrators.type='f'";
	$flux_only=1;
    } elsif ($ARGV[$i] eq "-C"){
	# only calibrate non-flux calibrators
	push @get_query_constraints,"calibrators.type='p'";
    } elsif ($ARGV[$i] eq "-cm"){
	# only calibrate cm calibrators
	push @get_query_constraints,"10000>any(observation.if_freq)";
	$cm_obs_only=1;
    } elsif ($ARGV[$i] eq "-mm"){
	# only calibrate mm calibrators
	push @get_query_constraints,"10000<any(observation.if_freq)";
	$mm_obs_only=1;
    } elsif ($ARGV[$i] eq "-L"){
	# change the limit
	$i++;
	$get_query_limit=$ARGV[$i];
#    } elsif ($ARGV[$i] eq "-R"){
#	# allow reprocessing, ie. include already processed calibrators
#	for (my $j=0;$j<=$#get_query_constraints;$j++){
#	    if ($get_query_constraints[$j] eq "observation.num_processed=0"){
#		$get_query_constraints[$j]="";
#		last;
#	    }
#	}
    } elsif ($ARGV[$i] eq "-W"){
	# change the working directory
	$i++;
	$working_directory=$ARGV[$i];
    } elsif ($ARGV[$i] eq "-A"){
	# set the minimum number of antennas on-source
	$i++;
	$minimum_ants=$ARGV[$i];
    } elsif ($ARGV[$i] eq "-T"){
	# set the minimum time on-source
	$i++;
	$minimum_time=$ARGV[$i];
    } elsif ($ARGV[$i] eq "-d"){
	# don't delete the final processed files
	$delete_process=0;
    } elsif ($ARGV[$i] eq "-B"){
	# do bandpass calibrators only
	push @get_query_constraints,"calibrators.type!='f'";
	$bandpass_only=1;
    } elsif ($ARGV[$i] eq "-R"){
	# go through and do the bandpass calibrators again, but this
	# time flux calibrate them
	$redo_bandpass=1;
    } elsif ($ARGV[$i] eq "-Q"){
	# the user wants to add their own SQL constraint
	$i++;
	push @get_query_constraints,$ARGV[$i];
    } elsif ($ARGV[$i] eq "-P"){
	# do phase calibrators only
	push @get_query_constraints,"calibrators.type!='f'";
	$phase_only=1;
    } elsif ($ARGV[$i] eq "-S"){
	# stop after this many sources have been successfully calibrated
	$i++;
	$stop_after=$ARGV[$i];
    }
}

if (($redo_bandpass==1)&&($bandpass_only==0)){
    # these options don't make sense
    print "cannot ask to redo bandpass calibrators without specifying bandpass only\n";
    exit;
} elsif ($redo_bandpass==1){
    # remove the restriction that requires unprocessed sources
    for (my $i=0;$i<=$#get_query_constraints;$i++){
	if ($get_query_constraints[$i] eq "observation.num_processed=0"){
	    $get_query_constraints[$i]="observation.num_processed>0";
	}
    }
}

if ($bandpass_only==1){
    # we wait until now so that we only select the appropriate wavelengths
    if ($cm_obs_only){
	push @get_query_constraints,
	"((calibrators.bandpass_21cm=true AND ".$highlow_21cm[0]."<any(observation.if_freq) AND ".
	    $highlow_21cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_13cm=true AND ".$highlow_13cm[0]."<any(observation.if_freq) AND ".
	    $highlow_13cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_6cm=true AND ".$highlow_6cm[0]."<any(observation.if_freq) AND ".
	    $highlow_6cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_3cm=true AND ".$highlow_3cm[0]."<any(observation.if_freq) AND ".
	    $highlow_3cm[1].">any(observation.if_freq)))";
    } elsif ($mm_obs_only) {
	push @get_query_constraints,
	"((calibrators.bandpass_12mm=true AND ".$highlow_12mm[0]."<any(observation.if_freq) AND ".
	    $highlow_12mm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_7mm=true AND ".$highlow_7mm[0]."<any(observation.if_freq) AND ".
	    $highlow_7mm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_3mm=true AND ".$highlow_3mm[0]."<any(observation.if_freq) AND ".
	    $highlow_3mm[1].">any(observation.if_freq)))";
    } else {
	push @get_query_constraints,
	"((calibrators.bandpass_21cm=true AND ".$highlow_21cm[0]."<any(observation.if_freq) AND ".
	    $highlow_21cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_13cm=true AND ".$highlow_13cm[0]."<any(observation.if_freq) AND ".
	    $highlow_13cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_6cm=true AND ".$highlow_6cm[0]."<any(observation.if_freq) AND ".
	    $highlow_6cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_3cm=true AND ".$highlow_3cm[0]."<any(observation.if_freq) AND ".
	    $highlow_3cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_12mm=true AND ".$highlow_12mm[0]."<any(observation.if_freq) AND ".
	    $highlow_12mm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_7mm=true AND ".$highlow_7mm[0]."<any(observation.if_freq) AND ".
	    $highlow_7mm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_3mm=true AND ".$highlow_3mm[0]."<any(observation.if_freq) AND ".
	    $highlow_3mm[1].">any(observation.if_freq)))";
    }
} elsif ($phase_only==1){
    # we wait until now so that we only select the appropriate wavelengths
    if ($cm_obs_only){
	push @get_query_constraints,
	"((calibrators.bandpass_21cm=false AND ".$highlow_21cm[0]."<any(observation.if_freq) AND ".
	    $highlow_21cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_13cm=false AND ".$highlow_13cm[0]."<any(observation.if_freq) AND ".
	    $highlow_13cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_6cm=false AND ".$highlow_6cm[0]."<any(observation.if_freq) AND ".
	    $highlow_6cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_3cm=false AND ".$highlow_3cm[0]."<any(observation.if_freq) AND ".
	    $highlow_3cm[1].">any(observation.if_freq)))";
    } elsif ($mm_obs_only){
	push @get_query_constraints,
	"((calibrators.bandpass_12mm=false AND ".$highlow_12mm[0]."<any(observation.if_freq) AND ".
	    $highlow_12mm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_7mm=false AND ".$highlow_7mm[0]."<any(observation.if_freq) AND ".
	    $highlow_7mm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_3mm=false AND ".$highlow_3mm[0]."<any(observation.if_freq) AND ".
	    $highlow_3mm[1].">any(observation.if_freq)))";
    } else {
	push @get_query_constraints,
	"((calibrators.bandpass_21cm=false AND ".$highlow_21cm[0]."<any(observation.if_freq) AND ".
	    $highlow_21cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_13cm=false AND ".$highlow_13cm[0]."<any(observation.if_freq) AND ".
	    $highlow_13cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_6cm=false AND ".$highlow_6cm[0]."<any(observation.if_freq) AND ".
	    $highlow_6cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_3cm=false AND ".$highlow_3cm[0]."<any(observation.if_freq) AND ".
	    $highlow_3cm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_12mm=false AND ".$highlow_12mm[0]."<any(observation.if_freq) AND ".
	    $highlow_12mm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_7mm=false AND ".$highlow_7mm[0]."<any(observation.if_freq) AND ".
	    $highlow_7mm[1].">any(observation.if_freq)) OR ".
	    "(calibrators.bandpass_3mm=false AND ".$highlow_3mm[0]."<any(observation.if_freq) AND ".
	    $highlow_3mm[1].">any(observation.if_freq)))";
    }
}

my $get_query="SELECT ".$get_query_parameters." FROM ".$get_query_source;
if ($#get_query_constraints>=0){
    $get_query.=" WHERE ";
    for (my $i=0;$i<=$#get_query_constraints;$i++){
	if ($get_query_constraints[$i] eq ""){
	    next;
	}
	if ($i>0){
	    $get_query.=" AND ";
	}
	$get_query.=$get_query_constraints[$i];
    }
}
if ($get_query_order ne ""){
    $get_query.=" ORDER BY ".$get_query_order;
}
if ($get_query_limit!=-1){
    $get_query.=" LIMIT ".$get_query_limit;
}
 
if ($output_debug>0){
    print "data selection query: $get_query\n";
}

# database details
my $dsn="dbi:Pg:dbname=atca;host=mentok;port=5432";
my $user="postgres";
my $password="";

# Setup the database connection
my $dbh=DBI->connect($dsn,$user,$password,{ RaiseError => 0, AutoCommit => 1});

# get the data
my $get=$dbh->prepare($get_query);
$get->execute;
my $n_successful=0;
while(my @get_response=$get->fetchrow){
    if ($output_debug>0){
	print "\n\n=== PROCESSING STARTS ===\n";
	print "source: ".$get_response[$observation{'source'}];
	print "   calibrator type: ".$get_response[$calibrators{'type'}];
	print "   id: ".$get_response[$observation{'id'}];
	print "   date: ".$get_response[$observation{'time'}];
	print "   project: ".$get_response[$observation{'project_code'}]."\n";
    }
    # what to do?
    if ($get_response[$calibrators{'type'}] eq "f"){
	# we have a flux calibrator, so all we need to do is calibrate it
	# and store it in the database
	if ($output_debug>0){
	    print "doing flux calibration procedure\n";
	}
    } elsif ($get_response[$calibrators{'type'}] eq "p"){
	# we have a phase calibrator
	if ($output_debug>0){
	    print "doing phase calibrator procedure\n";
	}
    }
	
    # check that it meets the minimum on-source antennae and time
    # constraints
    my @onsource_cycles=&postgresql_array_get($get_response[$observation{'onsource'}]);
    my @offsource_cycles=&postgresql_array_get($get_response[$observation{'offsource'}]);
    my $ants_onsource=0;
    my $reference_antenna=0;
    my $reference_cycles=0;
    for (my $i=0;$i<=$#onsource_cycles;$i++){
	if ($output_debug>2){
	    print "antenna ".($i+1).": $onsource_cycles[$i] cycles on-source\n";
	}
	if ($onsource_cycles[$i]>0){
	    $ants_onsource++;
	    if ($onsource_cycles[$i]>$reference_cycles){
		$reference_cycles=$onsource_cycles[$i];
		$reference_antenna=$i+1;
	    }
	}
    }
    if ($ants_onsource<$minimum_ants){
	if ($output_debug>0){
	    print "observation only has $ants_onsource antennae on-source; we will not".
		" reduce this data\n";
	}
	if ($ants_onsource<=$bad_ants){
	    if ($output_debug>0){
		print "observation has less than ".($bad_ants+1)." antennae on-source;".
		    " this observation will be marked as unusable\n";
	    }
	    &observation_set_unusable($dbh,$get_response[$observation{'id'}]);
	}
	next;
    }
    if ($output_debug>0){
	print "observation has $ants_onsource antennae on-source\n";
    }
    my ($mincycles,$maxcycles)=&minmax(@onsource_cycles);
    my @all_cycles;
    for (my $i=0;$i<=$#onsource_cycles;$i++){
	$all_cycles[$i]=$offsource_cycles[$i]+$onsource_cycles[$i];
    }
    my ($minallcycles,$maxallcycles)=&minmax(@all_cycles);
    my $max_obstime=$maxcycles*$get_response[$observation{'cycle_time'}];
    if ($max_obstime<$minimum_time){
	if ($output_debug>0){
	    print "observation has only $max_obstime seconds of data; we will not".
		" reduce this data\n";
	}
	&observation_set_unusable($dbh,$get_response[$observation{'id'}]);
	next;
    }
    if ($output_debug>0){
	print "observation has $max_obstime seconds of observation time\n";
    }
    my $max_totalobstime=$maxallcycles*$get_response[$observation{'cycle_time'}];
    if ($output_debug>0){
	print "total time on this scan is $max_totalobstime seconds\n";
    }

    # if we've been asked to redo the bandpass calibrators, does this
    # source already have a dataset that was processed by phase?
    my @proced_ids=&postgresql_array_get($get_response[$observation{'processed_ids'}]);
    my $nphaseproc=&has_phase_calibrated_dataset($dbh,@proced_ids);
    if ($nphaseproc>0){
	# this dataset has already been recalibrated, go to the next one
	if ($output_debug>0){
	    print "dataset has already been phase calibrated, skipping\n";
	}
	next;
    }

    # step 1: extract the data
    my $extracted_fits;
    if ($output_debug>0){
	print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	print "step 1. extract data\n\n";
    }
    my @redo_frequencies;
    if ($redo_bandpass==1){
	# we can just get the datasets from the database
	@redo_frequencies=&get_database_datasets($dbh,$working_directory,@proced_ids);
	if ($output_debug>1){
	    for (my $i=0;$i<=$#redo_frequencies;$i++){
		print "redoing frequency ".($i+1).": ".$redo_frequencies[$i]."\n";
	    }
	}
    } else {
	$extracted_fits=
	    &extract_rpfits($get_response[$observation{'rpfits_file'}],
			    $get_response[$observation{'header_number'}],
			    $get_response[$observation{'source'}],
			    $get_response[$observation{'time'}],$working_directory);
	
	if ($extracted_fits eq ""){
	    # couldn't get a file!
	    exit;
	}
    }
    
    # step 2: determine which IFs we want from this file
    if ($output_debug>0){
	print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	print "step 2. determine usable IFs\n\n";
    }
    my @if_bw=&postgresql_array_get($get_response[$observation{'if_bw'}]);
    my @if_freq=&postgresql_array_get($get_response[$observation{'if_freq'}]);
    my @if_nchans=&postgresql_array_get($get_response[$observation{'if_nchans'}]);
    my @usable_ifs;
    my @usable_if_freqs;
    my @usable_if_nchans;
    my @usable_if_bws;
    for (my $i=0;$i<=$#if_bw;$i++){
	# check that we have 2 GHz bandwidth
	if ($if_bw[$i]==2048){
	    # it's a continuum band
	    # check that we haven't already added an IF with the same frequency
	    my $already_added=0;
	    for (my $j=0;$j<=$#usable_ifs;$j++){
		if ($if_freq[$usable_ifs[$j]]==$if_freq[$i]){
		    # already got this frequency
		    $already_added=1;
		}
	    }
	    if ($already_added==0){
		# we have another usable IF
		push @usable_ifs,$i;
		push @usable_if_freqs,$if_freq[$i];
		push @usable_if_nchans,$if_nchans[$i];
		push @usable_if_bws,$if_bw[$i];
	    }
	}
    }
    if ($redo_bandpass==1){
	# remove any frequency that isn't in the list of available datasets
	for (my $i=0;$i<=$#usable_if_freqs;$i++){
	    my $keep=0;
	    for (my $j=0;$j<=$#redo_frequencies;$j++){
		if ($usable_if_freqs[$i]==$redo_frequencies[$j]){
		    $keep=1;
		    last;
		}
	    }
	    if ($keep==0){
		splice @usable_if_freqs,$i,1;
		splice @usable_if_nchans,$i,1;
		splice @usable_if_bws,$i,1;
		splice @usable_ifs,$i,1;
		$i--;
	    }
	}
    }
    if ($output_debug>0){
	print "found ".($#usable_ifs+1)." usable IFs:\n";
	for (my $i=0;$i<=$#usable_ifs;$i++){
	    print " IF ".($usable_ifs[$i]+1).": freq = ".$if_freq[$usable_ifs[$i]]."; bw = ".$if_bw[$usable_ifs[$i]].
		"; chans = ".$if_nchans[$usable_ifs[$i]]."\n";
	}
    }
    
    # only do the following steps if we're not redoing stuff
    my @bandpass_ids;
    my @bandpass_calibration;
    if ($redo_bandpass==0){
	# step 3: load the data into MIRIAD format
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 3. load RPFITS into MIRIAD format\n\n";
	}
	my $rc=&fits_to_miriad($extracted_fits,$get_response[$observation{'source'}],
			       $working_directory,@usable_ifs);
	# from this point, any capital letters in the source name will have been
	# converted to lowercase by the online system
	$get_response[$observation{'source'}]=~tr/A-Z/a-z/;
	if ($rc!=0){
	    # something went wrong, mark the observation as unusable
	    &observation_set_unusable($dbh,$get_response[$observation{'id'}]);
	    if ($delete_process==1){
		&clean_up($get_response[$observation{'source'}],$extracted_fits,
			  $working_directory,@usable_if_freqs);
	    }
	    next;
	}
	
	# step 4: flag based on calibration events
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 4. calibration event flagging\n\n";
	}
	&event_flag($dbh,$get_response[$observation{'source'}],
		    $get_response[$observation{'time'}],$max_totalobstime,
		    $get_response[$observation{'cycle_time'}],$working_directory,
		    @usable_if_freqs);
	
	for (my $i=0;$i<=$#usable_if_freqs;$i++){
	    $bandpass_calibration[$i]=1;
	}
    }

    # are we looking at a planet?
    my $planet=&is_planet($get_response[$observation{'source'}]);
    if ($output_debug>0){
	print $get_response[$observation{'source'}]." is ";
	if ($planet==0){
	    print "not ";
	}
	print "a planet\n";
    }

    if (($redo_bandpass==0)&&($phase_only==0)){
	# step 5: calibrate, round 1
	if ($planet==0){
	    # only calibrate if we're not dealing with a planet
	    if ($output_debug>0){
		print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
		print "step 5. calibration, round 1\n\n";
	    }
	    &calibrate($get_response[$observation{'source'}],
		       $get_response[$calibrators{'type'}],$reference_antenna,
		       $working_directory,$#usable_if_freqs,@bandpass_calibration,
		       @usable_if_freqs);
	    
	    # step 6: autoflag!
	    if ($output_debug>0){
		print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
		print "step 6. automatic RFI flagging\n\n";
	    }
	    &autoflag($get_response[$observation{'source'}],$working_directory,
		      @usable_if_freqs);
	}
    }

    # step 7: check that we still have data left
    if ($output_debug>0){
	print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	print "step 7. determine data flagging state\n\n";
    }
    my @flaglevels=&check_flagging($get_response[$observation{'source'}],$working_directory,
				   @usable_if_freqs);
    # and change the usable IF freqs accordingly
    my @original_usable_if_freqs=@usable_if_freqs;
    for (my $i=0;$i<=$#usable_if_freqs;$i++){
	if ($flaglevels[$i]==100.0){
	    # this IF has been completely flagged, so we drop it
	    splice @usable_if_freqs,$i,1;
	    splice @usable_if_nchans,$i,1;
	    splice @usable_if_bws,$i,1;
	    splice @usable_ifs,$i,1;
	    splice @flaglevels,$i,1;
	    splice @bandpass_calibration,$i,1;
	    $i--;
	}
    }
    # check if we have any IFs left
    if ($#usable_if_freqs<0){
	# nope
	if ($output_debug>0){
	    print "all IFs have been completely flagged...\n";
	}
	&observation_set_unusable($dbh,$get_response[$observation{'id'}]);
	if ($delete_process==1){
	    &clean_up($get_response[$observation{'source'}],$extracted_fits,
		      $working_directory,@original_usable_if_freqs);
	}
	next;
    }
    
    my @new_if_nchans;
    if ($redo_bandpass==0){
	# step 8: average the data
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 8. average data over channels\n\n";
	}
	@new_if_nchans=&average_data($get_response[$observation{'source'}],$working_directory,
				     ($#usable_if_freqs+1),@usable_if_freqs,@usable_if_nchans);
    } else {
	for (my $i=0;$i<=$#usable_if_nchans;$i++){
	    push @new_if_nchans,32;
	}
    }
    
    # step 9-pre(i): if we're doing phase calibration, or if we have a weak
    # flux cal, or a planet for a flux cal, search for a bandpass
    # calibrator observation which we can use
    my @fluxcal_ids;
    my $wavelength=&which_wavelength($if_freq[0]);
    if (($phase_only==1)||($planet==1)||($redo_bandpass==1)||
	(($flux_only==1)&&($get_response[$calibrators{"bandpass_".$wavelength}]==0))){
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 9-pre(i). find bandpass calibrator\n\n";
	}
	my @nearest_times=&find_nearest_events($dbh,$get_response[$observation{'time'}],
					       @if_freq);
	
	if (($phase_only==1)||($redo_bandpass==1)){
	    # get the id numbers of the appropriate flux calibrators
	    @fluxcal_ids=&find_nearest_fluxcal($dbh,$get_response[$observation{'time'}],
					       @nearest_times,$#if_freq,@if_freq,@usable_if_freqs);
	    # we have to have a flux calibrator to continue
	    for (my $i=0;$i<=$#fluxcal_ids;$i++){
		if ($fluxcal_ids[$i]==-1){
		    # we remove this frequency from the list, we really needed
		    # a flux calibration at this point
		    splice @usable_if_freqs,$i,1;
		    splice @usable_if_nchans,$i,1;
		    splice @usable_if_bws,$i,1;
		    splice @usable_ifs,$i,1;
		    splice @flaglevels,$i,1;
		    splice @bandpass_calibration,$i,1;
		    splice @bandpass_ids,$i,1;
		    splice @fluxcal_ids,$i,1;
		    $i--;
		}
	    }
	} else {
	    for (my $i=0;$i<=$#usable_if_freqs;$i++){
		$fluxcal_ids[$i]=-1;
	    }
	}

	if ($#usable_if_freqs>=0){
	    @bandpass_ids=&find_nearest_bandpasscal($dbh,$get_response[$observation{'time'}],
						    @nearest_times,$get_response[$observation{'source'}],
						    $#if_freq,@if_freq,$#usable_if_freqs,@fluxcal_ids,
						    @usable_if_freqs);
	    # now copy the bandpass calibrations if we have them
	    for (my $i=0;$i<=$#bandpass_ids;$i++){
		if ($bandpass_ids[$i]>0){
		    my $rc=&copy_bandpass($dbh,$bandpass_ids[$i],$get_response[$observation{'source'}],
					  $working_directory,$usable_if_freqs[$i]);
		    $bandpass_calibration[$i]=$rc;
		} else {
		    # remove this frequency from the list, without bandpass calibration
		    # we are nothing
		    splice @usable_if_freqs,$i,1;
		    splice @usable_if_nchans,$i,1;
		    splice @usable_if_bws,$i,1;
		    splice @usable_ifs,$i,1;
		    splice @flaglevels,$i,1;
		    splice @bandpass_calibration,$i,1;
		    splice @bandpass_ids,$i,1;
		    splice @fluxcal_ids,$i,1;
		    $i--;
		}
	    }
	}
    }

    # check we have data left
    if ($#usable_if_freqs<0){
	if ($output_debug>0){
	    print "unable to continue, no usable frequencies\n";
	}
	&observation_set_unusable($dbh,$get_response[$observation{'id'}]);
	if ($delete_process==1){
	    &clean_up($get_response[$observation{'source'}],$extracted_fits,
		      $working_directory,@original_usable_if_freqs);
	}
	next;
    }
	
    if ($planet==0){
	# step 9: calibrate, round 2
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 9. calibrate, round 2\n\n";
	}
	my $cal_success=&calibrate($get_response[$observation{'source'}],
				   $get_response[$calibrators{'type'}],$reference_antenna,
				   $working_directory,$#usable_if_freqs,
				   @bandpass_calibration,@usable_if_freqs);
	if ($cal_success<0){
	    # we haven't got a calibrated dataset
	    if ($output_debug>0){
		print "unable to calibrate dataset\n";
	    }
	    &observation_set_unusable($dbh,$get_response[$observation{'id'}]);
	    if ($delete_process==1){
		&clean_up($get_response[$observation{'source'}],$extracted_fits,
			  $working_directory,@original_usable_if_freqs);
	    }
	    next;
	}
    } elsif ($planet==1){
	# step 9: calibrate, round 2
	# for planets, this means running mfboot on them
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 9. calibrate, round 2\n\n";
	}
	my @planet_scaling=&scale_flux_to_planet($get_response[$observation{'source'}],
						 $working_directory,-1,@usable_if_freqs);
    }	
	
    # step 10: get all the extra data we need
    if ($redo_bandpass==0){
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 10. obtain extra metadata\n\n";
	}
	my @datrefs=&get_parameters($get_response[$observation{'source'}],
				    $extracted_fits,$working_directory,@usable_if_freqs);
	my %extradata=%{$datrefs[0]};
	if ($output_debug>1){
	    print "result of get_parameters:\n";
	    print "airtemp = ".$extradata{"airtemp_if$usable_if_freqs[0]"}."\n";
	    print "xtsys = ".&array_to_text(@{$extradata{"xtsys_if$usable_if_freqs[0]"}})."\n";
	}

	# step 11: put all this metadata information into the database now
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 11. update database\n\n";
	}
	&update_database_metadata($dbh,$get_response[$observation{'id'}],\%extradata,
				  ($#if_freq+1),($#usable_if_freqs+1),
				  @if_freq,@if_bw,@usable_if_freqs);
	
    }

    if (($get_response[$calibrators{'type'}] eq "p")&&
	($bandpass_only==0)){
	# we need to flux calibrate this observation
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 12-pre(i). flux calibration\n\n";
	}
#	# figure out when the closest calibration events to this observation are
#	my @nearest_times=&find_nearest_events($dbh,$get_response[$observation{'time'}],
#					       @if_freq);

	# check there is enough information to continue
	for (my $i=0;$i<=$#fluxcal_ids;$i++){
	    if ($fluxcal_ids[$i]==-1){
		# couldn't find a flux cal at this frequency
		splice @usable_if_freqs,$i,1;
		splice @usable_if_nchans,$i,1;
		splice @usable_if_bws,$i,1;
		splice @usable_ifs,$i,1;
		splice @fluxcal_ids,$i,1;
		splice @bandpass_ids,$i,1;
		splice @bandpass_calibration,$i,1;
		$i--;
	    }
	}
	if ($#usable_if_freqs<0){
	    # couldn't flux cal anything!
	    if ($output_debug>0){
		print "no usable flux calibrator observations could be found!\n";
	    }
	    &observation_set_unusable($dbh,$get_response[$observation{'id'}]);
	    if ($delete_process==1){
		&clean_up($get_response[$observation{'source'}],$extracted_fits,
			  $working_directory,@original_usable_if_freqs);
	    }
	    next;
	}
    }

    # we do the rest of the procedure one IF at a time
    my $num_added=0;
    for (my $i=0;$i<=$#usable_if_freqs;$i++){
	# step 12: make some flux and phase stability measurements
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 12. flux and phase measurements\n\n";
	}
	
	my $flux_scale_factor;
	my $bandpass_slope_factor;
	if (($get_response[$calibrators{'type'}] eq "p")&&
	    (($bandpass_only==0)||($redo_bandpass==1))){
	    # we need to flux calibrate this observation
	    if ($output_debug>0){
		print "adjusting flux scale...\n";
	    }
	    ($flux_scale_factor,$bandpass_slope_factor)=
		&adjust_flux_scale($dbh,$fluxcal_ids[$i],$get_response[$observation{'source'}],
				   $working_directory,$usable_if_freqs[$i]);
	    if ($output_debug>0){
		print "adjusted flux scale by a factor of $flux_scale_factor\n";
	    }
	    if ($flux_scale_factor==-999.999){
		# failed at flux scaling
		next;
	    }
	} elsif ($get_response[$calibrators{'type'}] eq "f"){
	    # we actually only need to correct for bandpass slope, so we
	    # correct like we're on a planet, even if we're not
	    ($flux_scale_factor,$bandpass_slope_factor)=
		&scale_flux_to_planet($get_response[$observation{'source'}],
				      $working_directory,-1,$usable_if_freqs[$i]);
	}
	
	my @measrefs=&make_measurements($get_response[$observation{'source'}],
					$working_directory,$usable_if_freqs[$i]);
	my %measurements=%{$measrefs[0]};
	
	# step 13: compress the calibrator dataset
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 13. archiving and compressing dataset\n\n";
	}
	&compress_dataset($get_response[$observation{'source'}],$working_directory,
			  $usable_if_freqs[$i]);
	
	# step 14: make a new processed observation item in the database, and insert
	# the values and data into it
	if ($output_debug>0){
	    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	    print "step 14. adding flux data to database\n\n";
	}
	my ($processed_as_flux,$processed_as_bandpass,$processed_as_phase);
	my ($fluxid,$bandid,$fluxscale,$bandslope);
	if ($get_response[$calibrators{'type'}] eq "f"){
	    $processed_as_flux=1;
	    $processed_as_bandpass=$bandpass_calibration[$i];
#	    $processed_as_bandpass=1;
	    $processed_as_phase=0;
	    $fluxscale=$flux_scale_factor;
	    $bandslope=$bandpass_slope_factor;
	    $fluxid=0;
	    if ($bandpass_calibration[$i]==0){
		$bandid=$bandpass_ids[$i];
	    } else {
		$bandid=0;
	    }
	} elsif ($bandpass_only==1){
	    $processed_as_flux=0;
	    $processed_as_bandpass=1;
	    if ($redo_bandpass==1){
		$processed_as_phase=1;
		$fluxscale=$flux_scale_factor;
		$bandslope=$bandpass_slope_factor;
		$fluxid=$fluxcal_ids[$i];
	    } else {
		$processed_as_phase=0;
		$fluxscale=0;
		$bandslope=0;
		$fluxid=0;
		$bandid=0;
	    }
#	    $fluxid=$fluxcal_ids[$i];
	} else {
	    $processed_as_flux=0;
	    if ($bandpass_calibration[$i]==1){
		$processed_as_bandpass=1;
		$processed_as_phase=0;
		$bandid=0;
	    } else {
		$processed_as_bandpass=0;
		$processed_as_phase=1;
		$bandid=$bandpass_ids[$i];
	    }
	    $fluxscale=$flux_scale_factor;
	    $bandslope=$bandpass_slope_factor;
	    $fluxid=$fluxcal_ids[$i];
	}
	my $new_fluxdata_id=
	    &new_fluxdata_entry($dbh,$get_response[$observation{'id'}],
				$get_response[$observation{'source'}],
				$usable_if_freqs[$i],$new_if_nchans[$i],
				$usable_if_bws[$i],($usable_ifs[$i]+1),
				$processed_as_flux,$processed_as_bandpass,
				$processed_as_phase,$fluxid,$bandid,
				$fluxscale,$bandslope,$working_directory,
				$redo_bandpass,\%measurements);
	
	if ($new_fluxdata_id>0){
	    # step 15: add this new item into the observation database as a processed
	    # observation
	    if ($output_debug>0){
		print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
		print "step 15. updating processed flags and links\n\n";
	    }
	    if ($redo_bandpass==0){
		&add_processed($dbh,$get_response[$observation{'id'}],$new_fluxdata_id);
	    }
	    $num_added++;

	    if ($planet==0){
		# step 16: check if the data is bad enough to warrant flagging it as
		# low quality
		if ($output_debug>0){
		    print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
		    print "step 16. checking data quality\n\n";
		}
		my $quality_flag=&data_quality_checks(\%measurements,$fluxscale);
		if ($quality_flag==1){
		    # the data is low quality
		    &mark_low_quality($dbh,$new_fluxdata_id);
		}
	    }
	}
    }
    
    # if no flux data entries were made, we mark this observation as unusable
    if ($num_added==0){
	&observation_set_unusable($dbh,$get_response[$observation{'id'}]);
    }
    
    # step 16: clean up
    if ($output_debug>0){
	print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n";
	print "step 16. clean up\n\n";
    }
    if ($delete_process==1){
	&clean_up($get_response[$observation{'source'}],$extracted_fits,
		  $working_directory,@original_usable_if_freqs);
    }

    # should we stop?
    $n_successful++;
    if ($stop_after>0){
	if ($n_successful>=$stop_after){
	    # yes we should stop
	    last;
	}
    }
}
$get->finish;

$dbh->disconnect;

sub copy_bandpass {
    my ($dbh,$bandpasscal_id,$source,$working_directory,$if_freq)=@_;

    # some MIRIAD commands
    my $miriad_gpcopy_command="gpcopy";

    # preparation
    $working_directory=~s/\/$//;
    my $returncode=0;
    
    my $database_server_login="postgres\@localhost";
    my $database_server_port=2222;
    my $database_server_path="/var/lib/postgresql";

    # first we extract the flux calibrator dataset
    my $bandpasscal_dataset_name="bandpasscaldata_".$bandpasscal_id.".tar.bz2";
    my $extracted_dataset=$database_server_path."/".$bandpasscal_dataset_name;
    my $gti_query="SELECT fluxdata.id,fluxdata.source,observation.time,fluxdata.frequency,".
	"lo_export(fluxdata.uv_data,'".$extracted_dataset."') FROM fluxdata,observation ".
	"WHERE fluxdata.id=".$bandpasscal_id." AND observation.id=ANY(fluxdata.observation_ids)";
    my $gti=$dbh->prepare($gti_query);
    $gti->execute;
    
    my $bandpasscal_uv;
    my $bandpasscal_source;
    while (my @gtiresults=$gti->fetchrow){
	my ($fluxdata_id,$fluxdata_source,$observation_time,$fluxdata_frequency,
	    $fluxdata_oid)=@gtiresults;
	if ($output_debug>0){
	    print "copying bandpass from source ".$fluxdata_source." \@ ".$fluxdata_frequency." MHz".
		" \@ ".$observation_time."\n";
	}
	$bandpasscal_uv=$fluxdata_source.".".$fluxdata_frequency;
	$bandpasscal_source=$fluxdata_source;
    }
    $gti->finish;

    # get the data back here
    my $scp_command="scp -P ".$database_server_port;
    $scp_command.=" ".$database_server_login.":".$extracted_dataset;
    $scp_command.=" ".$working_directory."/.";
    if ($output_debug>0){
	print "executing: $scp_command\n";
    }
    system $scp_command;
    # and remove the remote dataset
    my $srm_command="ssh -p ".$database_server_port;
    $srm_command.=" ".$database_server_login;
    $srm_command.=" rm -f ".$extracted_dataset;
    if ($output_debug>0){
	print "executing: $srm_command\n";
    }
    system $srm_command;

    # unarchive the data
    my $input_archive=$working_directory."/".$bandpasscal_dataset_name;
    my $untar_command="tar -C ".$working_directory;
    my $made_tmp_directory=0;
    if ($source eq $bandpasscal_source){
	# if we proceed, then we'll run into conflicting name
	# problems, so we make it go to a different directory
	if (!-e $working_directory."/tmp"){
	    my $mkdir_command="mkdir ".$working_directory."/tmp";
	    system $mkdir_command;
	    $made_tmp_directory=1;
	}
	$untar_command.="/tmp";
    }
    $untar_command.=" -xjf ".$input_archive;
    if ($output_debug>0){
	print "executing: $untar_command\n";
    }
    system $untar_command;
    # remove the archive
    my $lrm_command="rm -f ".$input_archive;
    if ($output_debug>0){
	print "executing: $lrm_command\n";
    }
    if ($delete_process==1){
	system $lrm_command;
    }

    # do the copying
    $working_directory=&shrink_path($working_directory);
    my $uncal_dataset=$working_directory."/".$source.".".$if_freq;
    my $bandpasscal_dataset=$working_directory."/";
    if ($source eq $bandpasscal_source){
	$bandpasscal_dataset.="tmp/";
    }
    $bandpasscal_dataset.=$bandpasscal_uv;

    if (!-e $bandpasscal_dataset){
	if ($output_debug>0){
	    print "bandpass calibrator dataset is not as expected!\n";
	    print "expected ".$bandpasscal_dataset."\n";
	}
	$returncode=1;
    } else {
	my $gpcopy_command=$miriad_gpcopy_command;
	$gpcopy_command.=" vis=".$bandpasscal_dataset;
	$gpcopy_command.=" out=".$uncal_dataset;
	if ($output_debug>0){
	    print "executing: $gpcopy_command\n";
	}
	system $gpcopy_command;

	# remove the bandpasscal dataset
	my $brm_command="rm -rf ".$bandpasscal_dataset;
	if ($output_debug>0){
	    print "executing: $brm_command\n";
	}
	if ($delete_process==1){
	    system $brm_command;
	}
	# remove the tmp directory if we made it
	if ($made_tmp_directory==1){
	    my $rdir_command="rmdir ".$working_directory."/tmp";
	    if ($delete_process==1){
		system $rdir_command;
	    }
	}
    }

    return $returncode;
}

sub mark_low_quality {
    my ($dbh,$fluxdata_id)=@_;

    my $lqf_query="UPDATE fluxdata SET quality_flag=false WHERE ".
	"id=".$fluxdata_id;
    if ($output_debug>0){
	print "executing query: $lqf_query\n";
    }
    my $lqf=$dbh->prepare($lqf_query);
    $lqf->execute;
    $lqf->finish;

}

sub data_quality_checks {
    my ($measptr,$fluxscale)=@_;
    
    my %measurements=%{$measptr};

    my $data_is_bad=0;

    # mark as bad if the flux uncertainty is a large fraction of the actual
    # flux value
    my $acceptable_flux_uncertainty=0.5;
    my @tripleamp_value=@{$measurements{'tripleamp_value'}};
    my @tripleamp_scatter_actual=@{$measurements{'tripleamp_scatter_actual'}};
    if ($tripleamp_scatter_actual[0]>=($tripleamp_value[0]*$acceptable_flux_uncertainty)){
	if ($output_debug>0){
	    print "data has too large a flux uncertainty!\n";
	}
	$data_is_bad=1;
    }
    
    # mark as bad if the uncertainty is exactly 0 (which is not possible)
    if ($tripleamp_scatter_actual[0]==0){
	if ($output_debug>0){
	    print "data has non-realistic flux uncertainty!\n";
	}
	$data_is_bad=1;
    }

    # mark as bad if the flux is negative or zero
    if ($tripleamp_value[0]<=0){
	if ($output_debug>0){
	    print "data has zero or negative flux!\n";
	}
	$data_is_bad=1;
    }

    # mark as bad if the flux scaling factor is out of range
    if ($fluxscale!=0){ # 0 means the flux wasn't scaled
	my ($min_fluxscale,$max_fluxscale)=(0.5,3.0);
	if (($fluxscale<$min_fluxscale)||($fluxscale>$max_fluxscale)){
	    if ($output_debug>0){
		print "data has an unacceptable flux scaling factor!\n";
	    }
	    $data_is_bad=1;
	}
    }

    # mark as bad if the measured amplitude scatter is much much greater
    # than the theoretical amplitude scatter
    my $bad_amp_scatter_factor=100.0;
    my @tripleamp_scatter_theoretical=@{$measurements{'tripleamp_scatter_theoretical'}};
    if ($tripleamp_scatter_actual[0]>=($tripleamp_scatter_theoretical[0]*
				       $bad_amp_scatter_factor)){
	if ($output_debug>0){
	    print "data should have lower flux uncertainty!\n";
	}
	$data_is_bad=1;
    }

    # mark as bad if the measured closure phase scatter is much much
    # greater than the theoretical closure phase scatter
#    my $bad_phase_scatter_factor=20.0;
#    my @closurephase_actual=@{$measurements{'closure_phases_actual'}};
#    my @closurephase_theoretical=@{$measurements{'closure_phases_theoretical'}};
#    if ($closurephase_actual[0]>=($closurephase_theoretical[0]*
#				  $bad_phase_scatter_factor)){
#	if ($output_debug>0){
#	    print "data should have lower closure phase scatter!\n";
#	}
#	$data_is_bad=1;
#    }

    # mark as bad if the combination of closure phase scatter ratio and
    # amplitude scatter ratio is greater than some acceptable cutoff
#    my $ratio_cutoff=25.0;
#    my $calculated_ratio=sqrt(($tripleamp_scatter_actual[0]/
#			       $tripleamp_scatter_theoretical[0])**2+
#			      ($closurephase_actual[0]/
#			       $closurephase_theoretical[0])**2);
#    if ($calculated_ratio>=$ratio_cutoff){
#	if ($output_debug>0){
#	    print "data has bad flux uncertainties and closure phase scatter!\n";
#	}
#	$data_is_bad=1;
#    }

    return $data_is_bad;
}

sub adjust_flux_scale {
    my ($dbh,$fluxcal_id,$source,$working_directory,$if_freq)=@_;
    
    # some MIRIAD commands
    my $miriad_gpboot_command="gpboot";
    my $miriad_mfboot_command="mfboot";

    # preparation
    $working_directory=~s/\/$//;
    $working_directory=&shrink_path($working_directory);
    my $flux_scale_factor=-999.999;
    my $bandpass_slope_factor=-999.999;

    my $database_server_login="postgres\@localhost";
    my $database_server_port=2222;
    my $database_server_path="/var/lib/postgresql";

    # first we extract the flux calibrator dataset
    my $fluxcal_dataset_name="fluxcaldata_".$fluxcal_id.".tar.bz2";
    my $extracted_dataset=$database_server_path."/".$fluxcal_dataset_name;
    my $gti_query="SELECT fluxdata.id,fluxdata.source,observation.time,fluxdata.frequency,".
	"lo_export(fluxdata.uv_data,'".$extracted_dataset."') FROM fluxdata,observation ".
	"WHERE fluxdata.id=".$fluxcal_id." AND observation.id=ANY(fluxdata.observation_ids)";
    if ($output_debug>0){
	print "executing query: $gti_query\n";
    }
    my $gti=$dbh->prepare($gti_query);
    $gti->execute;
    
    my $fluxcal_uv;
    my $flux_planet;
    my $flux_source;
    while (my @gtiresults=$gti->fetchrow){
	my ($fluxdata_id,$fluxdata_source,$observation_time,$fluxdata_frequency,
	    $fluxdata_oid)=@gtiresults;
	if ($output_debug>0){
	    print "scaling flux using source ".$fluxdata_source." \@ ".$fluxdata_frequency." MHz".
		" \@ ".$observation_time."\n";
	}
	$fluxcal_uv=$fluxdata_source.".".$fluxdata_frequency;
	$flux_planet=&is_planet($fluxdata_source);
	$flux_source=$fluxdata_source;
    }
    $gti->finish;

    # get the data back here
    my $scp_command="scp -P ".$database_server_port;
    $scp_command.=" ".$database_server_login.":".$extracted_dataset;
    $scp_command.=" ".$working_directory."/.";
    if ($output_debug>0){
	print "executing: $scp_command\n";
    }
    system $scp_command;
    # and remove the remote dataset
    my $srm_command="ssh -p ".$database_server_port;
    $srm_command.=" ".$database_server_login;
    $srm_command.=" rm -f ".$extracted_dataset;
    if ($srm_command!~/\/$/){
	if ($output_debug>0){
	    print "executing: $srm_command\n";
	}
	system $srm_command;
    }

    # unarchive the data
    my $input_archive=$working_directory."/".$fluxcal_dataset_name;
    my $untar_command="tar -C ".$working_directory;
    $untar_command.=" -xjf ".$input_archive;
    if ($output_debug>0){
	print "executing: $untar_command\n";
    }
    system $untar_command;
    # remove the archive
    my $lrm_command="rm -f ".$input_archive;
    if ($lrm_command!~/\/$/){
	if ($output_debug>0){
	    print "executing: $lrm_command\n";
	}
	if ($delete_process==1){
	    system $lrm_command;
	}
    }
    
    # do the scaling
    my $uncal_dataset=$working_directory."/".$source.".".$if_freq;
    my $fluxcal_dataset=$working_directory."/".$fluxcal_uv;
    if (!-e $fluxcal_dataset){
	if ($output_debug>0){
	    print "flux calibrator dataset is not as expected!\n";
	    print "expected ".$fluxcal_dataset."\n";
	}
    } else {
	if ($flux_planet==0){
	    # scale by flux of a known non-planet calibrator
	    my $gpboot_command=$miriad_gpboot_command;
	    $gpboot_command.=" vis=".$uncal_dataset;
	    $gpboot_command.=" cal=".$fluxcal_dataset;
	    if ($output_debug>0){
		print "executing: $gpboot_command\n";
	    }
	    open(GPBOOT,"-|")||exec $gpboot_command." 2>&1";
	    while(<GPBOOT>){
		chomp;
		my $line=$_;
		if ($output_debug>1){
		    print "MIRIAD: $line\n";
		}
		if ($line=~/^Secondary\sflux\sdensity\sscaled\sby\:/){
		    my @els=split(/\s+/,$line);
		    $flux_scale_factor=$els[$#els];
		}
	    }
	    close(GPBOOT);
	} else {
	    # set the scales the same between the planet and the
	    # calibrator we're using
	    my $gpboot_command=$miriad_gpboot_command;
	    $gpboot_command.=" vis=".$fluxcal_dataset;
	    $gpboot_command.=" cal=".$uncal_dataset;
	    if ($output_debug>0){
		print "executing: $gpboot_command\n";
	    }
	    system $gpboot_command;
	    $flux_scale_factor=1.0;
	}
	# now adjust the slope and the fluxscale if the flux calibrator
	# is a planet
	my $temp_scale_factor;
	($temp_scale_factor,$bandpass_slope_factor)=
	    &scale_flux_to_planet($flux_source,$working_directory,
				  0,$source,$if_freq);
	$flux_scale_factor*=$temp_scale_factor;
	if ($output_debug>1){
	    print "final flux adjustment: ".$flux_scale_factor."\n";
	    print "bandpass slope adjustment: ".$bandpass_slope_factor."\n";
	}
	# remove the fluxcal dataset
	my $frm_command="rm -rf ".$fluxcal_dataset;
	if ($frm_command!~/\/$/){
	    if ($output_debug>0){
		print "executing: $frm_command\n";
	    }
	    if ($delete_process==1){
		system $frm_command;
	    }
	}
    }

    return ($flux_scale_factor,$bandpass_slope_factor);
}

sub find_nearest_events {
    my ($dbh,$time,@ifs)=@_;

    my $time_difference="event_time-'".$time."'";
    my ($nearest_before,$nearest_after);
    my $evf_query="SELECT * FROM events WHERE (".
	"((event_type='dcal' OR event_type='acal') AND if_freq='".
	&array_to_text(@ifs)."')) AND EXTRACT(epoch FROM (".$time_difference."))<0".
	" ORDER BY abs(EXTRACT(epoch FROM (".$time_difference."))) LIMIT 1";
    if ($output_debug>0){
	print "executing query: $evf_query\n";
    }
    my $evf=$dbh->prepare($evf_query);
    $evf->execute;

    while(my @evf_response=$evf->fetchrow){
	$nearest_before=$evf_response[1];
    }
    $evf->finish;

    my $evf_query="SELECT * FROM events WHERE (".
	"((event_type='dcal' OR event_type='acal') AND if_freq='".
	&array_to_text(@ifs)."')) AND EXTRACT(epoch FROM (".$time_difference."))>0".
	" ORDER BY abs(EXTRACT(epoch FROM (".$time_difference."))) LIMIT 1";
    if ($output_debug>0){
	print "executing query: $evf_query\n";
    }
    my $evf=$dbh->prepare($evf_query);
    $evf->execute;

    while(my @evf_response=$evf->fetchrow){
	$nearest_after=$evf_response[1];
    }
    $evf->finish;

    return ($nearest_before,$nearest_after);
}

sub find_nearest_bandpasscal {
    my ($dbh,$time,$nearevent_before,$nearevent_after,$sourcecheck,$nif_freq,@ifs_stuff)=@_;
    my @ifs_config;
    for (my $i=0;$i<=$nif_freq;$i++){
	push @ifs_config,shift @ifs_stuff;
    }
    my $nuif_freq=shift @ifs_stuff;
    my @fluxcal_ids;
    for (my $i=0;$i<=$nuif_freq;$i++){
	push @fluxcal_ids,shift @ifs_stuff;
    }
    my @ifs=@ifs_stuff;

    my $unixtime_reference=&time_to_unixtime($time);
    my @bandpasscal_ids;

    for (my $i=0;$i<=$#ifs;$i++){
	# if we have a flux calibrator observation, we must use either it, or its
	# bandpass calibrator to ensure that the calibration solution for spectral
	# index (at least) is consistent
	my $found_good_id=-1;
	if ($fluxcal_ids[$i]!=-1){
	    my $wavelength=&which_wavelength($ifs[$i]);
	    my $det_query="SELECT fluxdata.bandpasscal_id,calibrators.type,calibrators.bandpass_".
		$wavelength." FROM fluxdata,calibrators WHERE ".
		"fluxdata.id=".$fluxcal_ids[$i];
	    if ($output_debug>0){
		print "executing query: $det_query\n";
	    }
	    my $det=$dbh->prepare($det_query);
	    $det->execute;
	    while (my @det_response=$det->fetchrow){
		if ($det_response[0]>0){
		    # there was a bandpass calibrator for this observation
		    $found_good_id=$det_response[0];
		} else {
		    # the flux calibrator should be used as the bandpass
		    # calibrator
		    $found_good_id=$fluxcal_ids[$i];
		}
	    }
	    $det->finish;
	}
	if ($found_good_id==-1){
	    # find an observation with the correct IF configuration nearest to our
	    # observation time that also falls between the known nearby event times, that
	    # also has some processed data
	    my $endtime="observation.time+totaltime(maxcycles(observation.onsource),".
		"maxcycles(observation.offsource),observation.cycle_time)";
	    my $ner_query="SELECT observation.id,observation.source,observation.if_freq,".
		"observation.num_processed,observation.processed_ids,".$endtime.
		",fluxdata.id,fluxdata.frequency,fluxdata.quality_flag ".
		"FROM observation,calibrators,fluxdata WHERE observation.source=".
		"calibrators.name AND fluxdata.processed_as_bandpass=true AND observation.if_freq='".
		&array_to_text(@ifs_config)."' AND observation.num_processed>0 AND ";
	    if ($nearevent_before ne ""){
		$ner_query.="(".$endtime.")>'".$nearevent_before."' AND ";
	    }
	    if ($nearevent_after ne ""){
		$ner_query.="(".$endtime.")<'".$nearevent_after."' AND ";
	    }
	    $ner_query.="fluxdata.id=any(observation.processed_ids) AND fluxdata.frequency=".
		$ifs[$i]." AND fluxdata.quality_flag=TRUE AND observation.source!='".$sourcecheck."' ".
		"ORDER BY abs(EXTRACT(epoch FROM ((".
		$endtime.")-'".$time."'))) LIMIT 1";
	    if ($output_debug>0){
		print "executing query: $ner_query\n";
	    }
	    my $ner=$dbh->prepare($ner_query);
	    $ner->execute;
	    while(my @ner_response=$ner->fetchrow){
		$found_good_id=$ner_response[6];
		if ($output_debug>0){
		    print "found usable bandpass calibration, using source ".
			$ner_response[1].", ending at time ".$ner_response[5].", frequency ".
			$ner_response[7]." MHz\n";
		}
	    }
	    $ner->finish;
	}
	$bandpasscal_ids[$i]=$found_good_id;
    }

    return (@bandpasscal_ids);
}

sub find_nearest_fluxcal {
    my ($dbh,$time,$nearevent_before,$nearevent_after,$nif_freq,@ifs_stuff)=@_;
    my @ifs_config;
    for (my $i=0;$i<=$nif_freq;$i++){
	push @ifs_config,shift @ifs_stuff;
    }
    my @ifs=@ifs_stuff;

    my $unixtime_reference=&time_to_unixtime($time);
    my @fluxcal_ids;


    for (my $i=0;$i<=$#ifs;$i++){
	# find an observation with the correct IF configuration nearest to our
	# observation time that also falls between the known nearby event times, that
	# also has some processed data
	my $endtime="observation.time+totaltime(maxcycles(observation.onsource),".
	    "maxcycles(observation.offsource),observation.cycle_time)";
	my $ner_query="SELECT observation.id,observation.source,observation.if_freq,".
	    "observation.num_processed,observation.processed_ids,".$endtime.
	    ",fluxdata.id,fluxdata.frequency,fluxdata.quality_flag ".
	    "FROM observation,calibrators,fluxdata WHERE observation.source=".
	    "calibrators.name AND calibrators.type='f' AND observation.if_freq='".
	    &array_to_text(@ifs_config)."' AND observation.num_processed>0 AND ";
	if ($nearevent_before ne ""){
	    $ner_query.="(".$endtime.")>'".$nearevent_before."' AND ";
	}
	if ($nearevent_after ne ""){
	    $ner_query.="(".$endtime.")<'".$nearevent_after."' AND ";
	}
	$ner_query.="fluxdata.id=any(observation.processed_ids) AND fluxdata.frequency=".
	    $ifs[$i]." AND fluxdata.quality_flag=TRUE AND (calibrators.low_freq*1000.0)<=observation.if_freq[1]".
	    " AND (calibrators.high_freq*1000.0)>=observation.if_freq[1] ORDER BY abs(EXTRACT(epoch FROM ((".
	    $endtime.")-'".$time."'))) LIMIT 1";
	if ($output_debug>0){
	    print "executing query: $ner_query\n";
	}
	my $ner=$dbh->prepare($ner_query);
	$ner->execute;
	my $found_good_id=-1;
	while(my @ner_response=$ner->fetchrow){
	    $found_good_id=$ner_response[6];
	    if ($output_debug>0){
		print "found usable flux calibration, using source ".
		    $ner_response[1].", ending at time ".$ner_response[5].", frequency ".
		    $ner_response[7]." MHz\n";
	    }
	}
	$ner->finish;
	$fluxcal_ids[$i]=$found_good_id;
    }

    return (@fluxcal_ids);
}

sub time_to_unixtime {
    my ($ptime)=@_;
    
    $ptime=~/^(\d{4})\-(\d{2})\-(\d{2})\s(\d{2})\:(\d{2})\:(\d{2})/;
    my ($year,$month,$day,$hour,$minute,$second)=($1,$2,$3,$4,$5,$6);
    $year-=1900;
    $month-=1;

    my $utime=mktime($second,$minute,$hour,$day,$month,$year,0,0);

    return $utime;
}

sub add_processed {
    my ($dbh,$obsid,$procid)=@_;

    # figure out how many processed items are already present
    my $gen_query="SELECT num_processed FROM observation WHERE id=".$obsid;
    if ($output_debug>0){
	print "executing query: $gen_query\n";
    }
    my $gen=$dbh->prepare($gen_query);
    $gen->execute;
    my $num_processed;
    while (my ($procres)=$gen->fetchrow){
	$num_processed=$procres;
    }
    $gen->finish;

    # one more now!
    $num_processed++;
    my $upd_query="UPDATE observation SET (num_processed,processed_ids[".$num_processed."])".
	" = (".$num_processed.",".$procid.") WHERE id=".$obsid;
    if ($output_debug>0){
	print "executing query: $upd_query\n";
    }
    my $upd=$dbh->prepare($upd_query);
    $upd->execute;
    $upd->finish;
}

sub new_fluxdata_entry {
    my ($dbh,$id,$source,$freq,$nchan,$bw,$ifnum,$fluxproc,$bandproc,$phaseproc,
	$fluxid,$bandid,$fluxscale,$bandslope,$working_directory,$overwrite,$measurementsptr)=@_;
    
    # preparation
    $working_directory=~s/\/$//;
    my %measurements=%{$measurementsptr};

    my $dat_query;
    my $overwrite_id;
    # are we overwriting an existing flux?
    if ($overwrite==1){
	# yes, but we have to find out which one
	my $fid_query="SELECT fluxdata.id,fluxdata.frequency,fluxdata.observation_ids FROM fluxdata ".
	    "WHERE ".$id."=ANY(fluxdata.observation_ids)";
	if ($output_debug>1){
	    print "executing query: $fid_query\n";
	}
	my $fid=$dbh->prepare($fid_query);
	$fid->execute;
	while (my @fid_response=$fid->fetchrow){
	    my @all_obsids=&postgresql_array_get($fid_response[2]);
	    if (($#all_obsids>0)||($all_obsids[0]!=$id)){
		# this flux data point does not come from the observation we want
		# or comes from more than just one observation
		next;
	    }
	    if ($fid_response[1]!=$freq){
		# not the right frequency
		next;
	    }
	    # this is the data point we want to overwrite
	    $overwrite_id=$fid_response[0];
	    last;
	}
	$fid->finish;
	# start the overwrite query
	$dat_query="UPDATE fluxdata SET ";
    } else {
	$dat_query="INSERT INTO fluxdata ";
    }
    $dat_query.="(num_observations,observation_ids,source,".
	"frequency,nchans,bandwidth,if_num,processed_as_flux,processed_as_bandpass,".
	"processed_as_phase,fluxcal_id,bandpasscal_id,flux_scale_factor,bandpass_slope_factor,data_freq,".
	"data_bw,tripleamp_value,tripleamp_scatter_measured,tripleamp_scatter_theoretical,".
	"closurephase_measured,closurephase_theoretical,uv_data)";
    if ($overwrite==1){
	$dat_query.="=";
    } else {
	$dat_query.=" VALUES ";
    }
    $dat_query.="(";
    $dat_query.="1,'{".$id."}','".$source."',".$freq.",".$nchan.",".$bw.",".$ifnum.",";
    if ($fluxproc==1){
	$dat_query.="TRUE";
    } else {
	$dat_query.="FALSE";
    }
    $dat_query.=",";
    if ($bandproc==1){
	$dat_query.="TRUE";
    } else {
	$dat_query.="FALSE";
    }
    $dat_query.=",";
    if ($phaseproc==1){
	$dat_query.="TRUE";
    } else {
	$dat_query.="FALSE";
    }
    $dat_query.=",";
    if (($fluxproc==1)||
	(($bandproc==1)&&($phaseproc==0))){
	$dat_query.="NULL";
    } else {
	$dat_query.=$fluxid;
    }
    $dat_query.=",";
    if ($bandproc==1){
	$dat_query.="NULL";
    } else {
	$dat_query.=$bandid;
    }
    $dat_query.=",";
    if ($fluxscale eq ""){
	$fluxscale=-999.999;
    }
    if ($bandslope eq ""){
	$bandslope=-999.999;
    }
    if (($bandproc==1)&&($phaseproc==0)){
	$dat_query.="NULL";
    } else {
	$dat_query.=$fluxscale;
    }
    $dat_query.=",";
    $dat_query.=$bandslope;
    $dat_query.=",";
    $dat_query.="'".&array_to_text(@{$measurements{'frequencies'}})."','".
	&array_to_text(@{$measurements{'bandwidths'}})."','".
	&array_to_text(@{$measurements{'tripleamp_value'}})."','".
	&array_to_text(@{$measurements{'tripleamp_scatter_actual'}})."','".
	&array_to_text(@{$measurements{'tripleamp_scatter_theoretical'}})."','".
	&array_to_text(@{$measurements{'closure_phases_actual'}})."','".
	&array_to_text(@{$measurements{'closure_phases_theoretical'}})."',";

    # check if there are any bad values
    if (($dat_query=~/\'nan\'/)||($dat_query=~/\*+/)){
	# yep, this query won't work!
	return -1;
    }
    
    # copy the compressed dataset to the database server
    my $input_dataset=$working_directory."/".$source.".".$freq.".tar.bz2";
    my $database_server_login="postgres\@localhost";
    my $database_server_port=2222;
    my $database_server_path="/var/lib/postgresql";
    my $copy_command="scp -P ".$database_server_port;
    $copy_command.=" ".$input_dataset;
    $copy_command.=" ".$database_server_login.":.";
    if ($output_debug>0){
	print "executing: $copy_command\n";
    }
    system $copy_command;
    $dat_query.="lo_import('".$database_server_path."/".$source.".".$freq.".tar.bz2')";
    $dat_query.=")";
    if ($overwrite==1){
	$dat_query.=" WHERE id=".$overwrite_id;
    }

    if ($output_debug>0){
	print "executing query: $dat_query\n";
    }
    my $dat=$dbh->prepare($dat_query);
    my $rc=$dat->execute;
    if ($rc!=1){
	# we may have gotten an error!
	if ($dat->err!=0){
	    if ($output_debug>0){
		print "error inserting into database!\n";
	    }
	    $dat->finish;
	    return -1;
	}
    }
    $dat->finish;
    # clean up the compressed dataset
    my $remove_command="ssh -p ".$database_server_port;
    $remove_command.=" ".$database_server_login;
    $remove_command.=" rm -f ".$source.".".$freq.".tar.bz2";
    if ($remove_command!~/\/$/){
	if ($output_debug>0){
	    print "executing: $remove_command\n";
	}
	system $remove_command;
    }

    # now determine what id that entry has
    my $entry_id;
    if ($overwrite==0){
	my $gid_query="SELECT id FROM fluxdata ORDER BY id DESC LIMIT 1";
	if ($output_debug>0){
	    print "executing query: $gid_query\n";
	}
	my $gid=$dbh->prepare($gid_query);
	$gid->execute;
	while (my ($idresponse)=$gid->fetchrow){
	    $entry_id=$idresponse;
	}
	$gid->finish;
    } else {
	$entry_id=$overwrite_id;
    }

    return $entry_id;
}

sub make_measurements {
    my ($source,$working_directory,$if_freq)=@_;
    
    # some MIRIAD commands
    my $miriad_closure_command="closure";

    # preparation
    $working_directory=~s/\/$//;

    my %measurements;

    # get the frequencies of each channel
    my @freqs=&frequency_axis($source,$working_directory,$if_freq);
    my $avchans=$#freqs;
    if ($output_debug>0){
	print "got $avchans channels in averaged dataset, frequencies:\n";
	print &array_to_text(@freqs)."\n";
    }
    my $n_spectral_measurements=4; # make this many measurements across the band
    my $spectral_channels=$avchans/$n_spectral_measurements;

    my @first_channels;
    my @last_channels;
    my @middle_channels;
    my @middle_freqs;
    my @bandwidths;
    for (my $j=0;$j<=$n_spectral_measurements;$j++){
	if ($j==0){
	    # special case, the whole band
	    $first_channels[$j]=1;
	    $last_channels[$j]=$#freqs;
	} else {
	    $first_channels[$j]=1+($j-1)*$spectral_channels;
	    $last_channels[$j]=$j*$spectral_channels;
	}
	$middle_channels[$j]=($first_channels[$j]+$last_channels[$j])/2.0;
	if ($middle_channels[$j]!=floor($middle_channels[$j])){
	    # between 2 channels
	    $middle_freqs[$j]=($freqs[floor($middle_channels[$j])]+
			       $freqs[ceil($middle_channels[$j])])/2.0;
	} else {
	    $middle_freqs[$j]=$freqs[$middle_channels[$j]];
	}
	$bandwidths[$j]=$freqs[$last_channels[$j]]-$freqs[$first_channels[$j]];
    }

    $measurements{'frequencies'}=[@middle_freqs];
    $measurements{'bandwidths'}=[@bandwidths];

    # get closure phases
    my @closure_phases_actual;
    my @closure_phases_theoretical;
    my $input_dataset=$working_directory."/".$source.".".$if_freq;
    for (my $i=0;$i<=$#first_channels;$i++){
	my $closure_command=$miriad_closure_command;
	$closure_command.=" vis=".$input_dataset;
	$closure_command.=" stokes=i";
	if ($i>0){
	    $closure_command.=" line=chan,$spectral_channels,".$first_channels[$i];
	}
	if ($output_debug>0){
	    print "executing $closure_command\n";
	}
	open(CLOSURE,"-|")||exec $closure_command." 2>&1";
	while(<CLOSURE>){
	    chomp;
	    my $line=$_;
	    if ($output_debug>2){
		print "MIRIAD: $line\n";
	    }
	    my @els=split(/\s+/,$line);
	    if ($line=~/^Actual\sclosure\sphase/){
		push @closure_phases_actual,$els[$#els];
	    } elsif ($line=~/^Theoretical\sclosure\sphase/){
		push @closure_phases_theoretical,$els[$#els];
	    } elsif ($line=~/^\#\#\#\sFatal\sError/){
		# this command has failed
		push @closure_phases_actual,-999.999;
		push @closure_phases_theoretical,-999.999;
	    }
	}
	close(CLOSURE);
    }
    $measurements{'closure_phases_actual'}=[@closure_phases_actual];
    $measurements{'closure_phases_theoretical'}=[@closure_phases_theoretical];

    # get triple amplitudes
    my @tripleamp_values;
    my @tripleamp_scatter_actual;
    my @tripleamp_scatter_theoretical;
    for (my $i=0;$i<=$#first_channels;$i++){
	my $closure_command=$miriad_closure_command;
	$closure_command.=" vis=".$input_dataset;
	$closure_command.=" stokes=i";
	$closure_command.=" options=amplitude";
	if ($i>0){
	    $closure_command.=" line=chan,$spectral_channels,".$first_channels[$i];
	}
	if ($output_debug>0){
	    print "executing $closure_command\n";
	}
	open(CLOSURE,"-|")||exec $closure_command." 2>&1";
	while(<CLOSURE>){
	    chomp;
	    my $line=$_;
	    if ($output_debug>2){
		print "MIRIAD: $line\n";
	    }
	    $line=~s/\:/\: /g;
	    my @els=split(/\s+/,$line);
	    if ($line=~/^Mean\samplitude/){
		push @tripleamp_values,$els[$#els];
	    } elsif ($line=~/^Actual\samplitude\srms/){
		push @tripleamp_scatter_actual,$els[$#els];
	    } elsif ($line=~/^Theoretical\samplitude\srms/){
		push @tripleamp_scatter_theoretical,$els[$#els];
	    } elsif ($line=~/^\#\#\#\sFatal\sError/){
		# this command has failed
		push @tripleamp_values,-999.999;
		push @tripleamp_scatter_actual,-999.999;
		push @tripleamp_scatter_theoretical,-999.999;
	    }
	}
	close(CLOSURE);
    }
    $measurements{'tripleamp_value'}=[@tripleamp_values];
    $measurements{'tripleamp_scatter_actual'}=[@tripleamp_scatter_actual];
    $measurements{'tripleamp_scatter_theoretical'}=[@tripleamp_scatter_theoretical];

    return (\%measurements);
}

sub frequency_axis {
    my ($source,$working_directory,$if_freq)=@_;

    # some MIRIAD commands
    my $miriad_uvlist_command="uvlist";
    
    # preparation
    $working_directory=~s/\/$//;

    my $input_dataset=$working_directory."/".$source.".".$if_freq;
    my $uvlist_command=$miriad_uvlist_command;
    $uvlist_command.=" vis=".$input_dataset;
    $uvlist_command.=" options=spectral";
    if ($output_debug>0){
	print "executing: $uvlist_command\n";
    }
    open(UVLIST,"-|")||exec $uvlist_command;
    my ($startchan,$numchan,$startfreq,$freqinterval);
    while(<UVLIST>){
	chomp;
	my $line=$_;
	my @els=split(/\s+/,$line);
	if ($line=~/^Start\schannel/){
	    $startchan=$els[$#els];
	} elsif ($line=~/^Number\sof\schannels/){
	    $numchan=$els[$#els];
	} elsif ($line=~/Start\sfrequency/){
	    $startfreq=$els[$#els];
	} elsif ($line=~/Frequency\sincrement/){
	    $freqinterval=$els[$#els];
	}
    }
    close(UVLIST);

    my @frequencies;
    for (my $i=$startchan;$i<=$numchan;$i++){
	$frequencies[$i]=$startfreq+($i-$startchan)*$freqinterval;
    }

    return @frequencies;
}

sub compress_dataset {
    my ($source,$working_directory,@ifs)=@_;
    
    # preparation
    $working_directory=~s/\/$//;

    for (my $i=0;$i<=$#ifs;$i++){
	my $input_dataset=$source.".".$ifs[$i];
	my $output_dataset=$working_directory."/".$input_dataset.".tar.bz2";
	my $compress_command="tar -C $working_directory -cjf";
	$compress_command.=" ".$output_dataset;
	$compress_command.=" ".$input_dataset;
	if ($output_debug>0){
	    print "executing: $compress_command\n";
	}
	system $compress_command;
    }
}

sub update_database_metadata {
    my ($dbh,$id,$extradataptr,$n_ifs,$n_usable,@ifs_stuff)=@_;

    my @all_if_freqs;
    my @all_if_bws;
    my @usable_if_freqs;
    for (my $i=0;$i<$n_ifs;$i++){
	push @all_if_freqs,shift @ifs_stuff;
    }
    for (my $i=0;$i<$n_ifs;$i++){
	push @all_if_bws,shift @ifs_stuff;
    }
    for (my $i=0;$i<$n_usable;$i++){
	push @usable_if_freqs,shift @ifs_stuff;
    }

    my %extradata=%{$extradataptr};

    my $sfx="_if".$usable_if_freqs[0];
    # the standard, always present (hopefully!) variables
    my $met_query_variables="az,el,attenmm,subrefl,corr_cfg,airtemp,chi,evector,jyperk,".
	"pntra,pntdec,relhumid,smonrms,wind,winddir";
    my $met_query_values=$extradata{"antaz$sfx"}.",".$extradata{"antel$sfx"}.",'{".
	&array_to_text(@{$extradata{'attenmm_a'}}).",".
	&array_to_text(@{$extradata{'attenmm_b'}})."}','".
	&array_to_text(@{$extradata{'subrefl'}})."','".$extradata{'corr_cfg'}."',".
	$extradata{"airtemp$sfx"}.",".$extradata{"chi$sfx"}.",".$extradata{"evector$sfx"}.",".
	$extradata{"jyperk$sfx"}.",".$extradata{"pntra$sfx"}.",".$extradata{"pntdec$sfx"}.",".
	$extradata{"relhumid$sfx"}.",".$extradata{"smonrms$sfx"}.",".$extradata{"wind$sfx"}.",".
	$extradata{"winddir$sfx"};
    # variables that may or may not be present
    my @maybe_present=("xtsys","ytsys","xyphase","xcaljy","ycaljy","xgtp","ygtp",
		       "xsdo","ysdo");
    for (my $i=0;$i<=$#maybe_present;$i++){
	if ($extradata{"$maybe_present[$i]$sfx"}){
	    my $tsys_string="'{";
	    for (my $j=0;$j<=$#all_if_freqs;$j++){
		if ($j!=0){
		    $tsys_string.=",";
		}
		if ($all_if_bws[$j]!=2048){
		    $tsys_string.="{";
		    for (my $k=0;$k<$extradata{"nants_if$all_if_freqs[$j]"};$k++){
			if ($k!=0){
			    $tsys_string.=",";
			}
			$tsys_string.="NULL";
		    }
		    $tsys_string.="}";
		} else {
		    my $added=0;
		    for (my $k=0;$k<=$#usable_if_freqs;$k++){
			my $subadded=1;
			if ($usable_if_freqs[$k]==$all_if_freqs[$j]){
			    my $tts=&array_to_text(@{$extradata{"$maybe_present[$i]\_if$all_if_freqs[$j]"}});
			    $added=1;
			    if ($tts eq "{}"){
				$subadded=0;
			    } else {
				$tsys_string.=$tts;
			    }
			}
			if ($subadded==0){
			    $tsys_string.="{";
			    for (my $l=0;$l<$extradata{"nants_if$all_if_freqs[$j]"};$l++){
				if ($l!=0){
				    $tsys_string.=",";
				}
				$tsys_string.="NULL";
			    }
			    $tsys_string.="}";
			    $added=1;
			}
		    }
		    if ($added==0){
			$tsys_string.="{";
			for (my $k=0;$k<$extradata{"nants_if$all_if_freqs[$j]"};$k++){
			    if ($k!=0){
				$tsys_string.=",";
			    }
			    $tsys_string.="NULL";
			}
			$tsys_string.="}";
		    }
		}
	    }
	    $tsys_string.="}'";
	    $met_query_variables.=",$maybe_present[$i]";
	    $met_query_values.=",".$tsys_string;
	}
    }
    if ($extradata{'point_time'}){
	$met_query_variables.=",pointcor,point_time,point_source,point_az,point_el";
	$met_query_values.=",'{".&array_to_text(@{$extradata{'pointcor_az'}}).",".
	    &array_to_text(@{$extradata{'pointcor_el'}})."}','".$extradata{'point_time'}."','".
	    $extradata{'point_source'}."',".$extradata{'point_az'}.",".
	    $extradata{'point_el'};
    }
    my $met_query="UPDATE observation SET (".$met_query_variables.")".
	"= (".$met_query_values.") WHERE id=".$id;
    # fix any {} problems
    $met_query=~s/\{\}/\{NULL\}/g;
    if ($output_debug>0){
	print "executing query: $met_query\n";
    }
    my $met=$dbh->prepare($met_query);
    $met->execute;
    $met->finish;
    
}

sub average_data {
    my ($source,$working_directory,$n_ifs,@ifs_stuff)=@_;
    
    my @if_freqs;
    my @if_nchans;
    for (my $i=0;$i<$n_ifs;$i++){
	push @if_freqs,shift @ifs_stuff;
    }
    for (my $i=0;$i<$n_ifs;$i++){
	push @if_nchans,shift @ifs_stuff;
    }

    # some MIRIAD commands
    my $miriad_uvaver_command="uvaver";

    # preparation
    $working_directory=~s/\/$//;
    
    # some parameters
    my $nchans=32; # the number of channels the output dataset should have
    my @new_if_nchans;

    my $current_directory=getcwd();
    chdir $working_directory;

    for (my $i=0;$i<=$#if_freqs;$i++){
	my $input_dataset=$source.".".$if_freqs[$i];
	my $output_dataset=$input_dataset.".aver";
	my $average_chans=($if_nchans[$i]-1)/$nchans;
	my $uvaver_line_spec="line=chan,".$nchans.",1,".$average_chans;
	my $uvaver_command=$miriad_uvaver_command;
	$uvaver_command.=" vis=".$input_dataset;
	$uvaver_command.=" out=".$output_dataset;
	$uvaver_command.=" ".$uvaver_line_spec;
	$uvaver_command.=" options=nopass,nopol,nocal";
	if ($output_debug>0){
	    print "executing: $uvaver_command\n";
	}
	system $uvaver_command;
	if ($delete_process==1){
	    my $remove_command="rm -rf ".$input_dataset;
	    if ($remove_command!~/\/$/){
		if ($output_debug>0){
		    print "executing: $remove_command\n";
		}
		system $remove_command;
	    }
	} else {
	    my $rename_command="mv ".$input_dataset." ".$input_dataset.".nonavg";
	    if ($output_debug>0){
		print "executing: $rename_command\n";
	    }
	    system $rename_command;
	}
	my $rename_command="mv ".$output_dataset." ".$input_dataset;
	if ($output_debug>0){
	    print "executing: $rename_command\n";
	}
	system $rename_command;
	push @new_if_nchans,$nchans;
    }

    chdir $current_directory;

    return @new_if_nchans;
}

sub clean_up {
    # deletes any files we've created
    my ($source,$rpfits,$working_directory,@ifs)=@_;

    # preparation
    $working_directory=~s/\/$//;
    
    # remove the MIRIAD files and compressed archives
    for (my $i=0;$i<=$#ifs;$i++){
	my $remove_command="rm -rf";
	$remove_command.=" ".$working_directory."/".$source.".".$ifs[$i];
	if ($remove_command!~/\/$/){
	    if ($output_debug>0){
		print "executing: $remove_command\n";
	    }
	    system $remove_command;
	}
	
	$remove_command.=".tar.bz2";
	if ($output_debug>0){
	    print "executing: $remove_command\n";
	}
	system $remove_command;
	
    }

    # remove the RPFITS file
    if ($rpfits ne ""){
	my $remove_command="rm -rf";
	$remove_command.=" ".$rpfits;
	if ($remove_command!~/\/$/){
	    if ($output_debug>0){
		print "executing: $remove_command\n";
	    }
	    system $remove_command;
	}
    }
    
}

sub check_flagging {
    my ($source,$working_directory,@ifs)=@_;

    # some MIRIAD commands
    my $miriad_uvfstats_command="uvfstats";
    
    # preparation
    $working_directory=~s/\/$//;

    my @flagging_levels;
    for (my $i=0;$i<=$#ifs;$i++){
	my $input_dataset=$working_directory."/".$source.".".$ifs[$i];
	my $uvfstats_command=$miriad_uvfstats_command;
	$uvfstats_command.=" vis=".$input_dataset;
	$uvfstats_command.=" \"select=pol(xx,yy)\"";
	$uvfstats_command.=" mode=overall";
	if ($output_debug>0){
	    print "executing: $uvfstats_command\n";
	}
	open(UVFSTATS,"-|")||exec $uvfstats_command;
	while(<UVFSTATS>){
	    chomp;
	    my $line=$_;
	    if ($line=~/Overall/){
		my $flevel=$line;
		$flevel=~s/\s*Overall\s+(.*)\%/$1/;
		$flagging_levels[$i]=$flevel;
	    }
	}
	close(UVFSTATS);
	if ($output_debug>0){
	    print "found $flagging_levels[$i] % flagged data in IF $ifs[$i]\n";
	}
    }

    return @flagging_levels;
}

sub postgresql_to_miriad_time {
    my ($postgresql_time)=@_;
    
    $postgresql_time=~/^([^\-]*)\-([^\-]*)\-([^\-]*)\s([^\:]*)\:([^\:]*)\:(.*)$/;
    my ($year,$month,$day,$hour,$minute,$second)=($1,$2,$3,$4,$5,$6);
    
    my $miriad_time=sprintf "%4d%3s%02d:%02d:%02d:%02d",$year,
    &month_number_to_name($month),$day,$hour,$minute,$second;
    
    return $miriad_time;
}

sub month_number_to_name {
    my ($month_number)=@_;
    
    my @month_names=("JAN","FEB","MAR","APR","MAY","JUN",
		     "JUL","AUG","SEP","OCT","NOV","DEC");
    $month_number--;
    return $month_names[$month_number];
    
}

sub event_flag {
    my ($dbh,$source,$start_time,$scan_time,$cycle_time,$working_directory,@ifs)=@_;

    # some MIRIAD commands
    my $miriad_uvflag_command="uvflag";

    # preparation
    $working_directory=~s/\/$//;
    $working_directory=&shrink_path($working_directory);

    # get the events that occurred during the scan
    my $evs_query="SELECT * FROM events WHERE event_time>'".$start_time."' AND ".
	"event_time<(TIMESTAMP '".$start_time."' + INTERVAL '".$scan_time." second') ".
	"ORDER BY event_time DESC LIMIT 1";
    if ($output_debug>0){
	print "executing query: $evs_query\n";
    }
    my $evs=$dbh->prepare($evs_query);
    $evs->execute;

    my $needs_flagging=0;
    my $event_time;
    while(my @events=$evs->fetchrow){
	# pretty much any correlator event will make us need to recalibrate, so
	# we flag all the way from the start to the time of the last event (which
	# should be the output of the query), plus four extra cycles to let the
	# event conclude.
	$needs_flagging=1;
	$event_time=$events[1];
    }
    $evs->finish;

    if ($needs_flagging==0){
	# no flagging required
	return;
    }

    my $buffer_time=4*$cycle_time; # in seconds
    # we ask postgresql for the time answers since it knows how to do it :)
    my $gtq_query="SELECT TIMESTAMP '".$event_time."' + INTERVAL '".$buffer_time." second'";
    if ($output_debug>1){
	print "executing query $gtq_query\n";
    }
    my $gtq=$dbh->prepare($gtq_query);
    $gtq->execute;
    
    my $final_time;
    while(my @times=$gtq->fetchrow){
	$final_time=$times[0];
    }
    $gtq->finish;

    # now flag the datasets
    for (my $i=0;$i<=$#ifs;$i++){
	my $input_dataset=$working_directory."/".$source.".".$ifs[$i];
	my $uvflag_command=$miriad_uvflag_command;
	$uvflag_command.=" vis=".$input_dataset;
	$uvflag_command.=" flagval=flag";
	$uvflag_command.=" \"select=time(".
	    &postgresql_to_miriad_time($start_time).",".
	    &postgresql_to_miriad_time($final_time).")\"";
	if ($output_debug>0){
	    print "executing $uvflag_command\n";
	}
	system $uvflag_command;
    }
    
}

sub scale_flux_to_planet {
    my ($planetname,$working_directory,$num_datasets,@if_stuff)=@_;
    my @datasets;
    for (my $i=0;$i<=$num_datasets;$i++){
	push @datasets,shift @if_stuff;
    }
    my @ifs=@if_stuff;

    if ($output_debug>1){
	print "got ".($#datasets+1)."datasets:\n";
	for (my $i=0;$i<=$#datasets;$i++){
	    print "  ".$datasets[$i]."\n";
	}
	print "got ".($#ifs+1)." IFs:\n";
	for (my $i=0;$i<=$#ifs;$i++){
	    print "  ".$ifs[$i]."\n";
	}
    }

    # some MIRIAD commands
    my $miriad_mfboot_command="mfboot";
    
    # preparation
    $working_directory=~s/\/$//;
    $working_directory=&shrink_path($working_directory);

    my @flux_scaling_factor;
    my @bandpass_slope_correction;
    
    for (my $i=0;$i<=$#ifs;$i++){
	my $mfboot_command=$miriad_mfboot_command;
	$mfboot_command.=" vis=";
	my $dataset=$working_directory."/".$planetname.".".
	    $ifs[$i];
	$mfboot_command.=$dataset;
	for (my $j=0;$j<=$#datasets;$j++){
	    my $thisdataset=$working_directory."/".
		$datasets[$j].".".$ifs[$i];
	    $mfboot_command.=",".$thisdataset;
	}
	$mfboot_command.=" \"select=source(".$planetname.")\"";
	if ($output_debug>0){
	    print "executing: $mfboot_command\n";
	}
	open(MFBOOT,"-|")||exec $mfboot_command;
	while(<MFBOOT>){
	    chomp;
	    my $line=$_;
	    if ($output_debug>1){
		print "MIRIAD: $line\n";
	    }
	    my @els=split(/\s+/,$line);
	    if ($line=~/^Scaling\sthe\sdata\sby/){
		push @flux_scaling_factor,$els[$#els];
	    } elsif ($line=~/^Adjusting\sspectral\sslope\sby/){
		push @bandpass_slope_correction,$els[$#els];
	    }
	}
	close(MFBOOT);
    }
   
    return (@flux_scaling_factor,@bandpass_slope_correction);
}

sub autoflag {
    my ($source,$working_directory,@ifs)=@_;

    # some MIRIAD commands
    my $miriad_mirflag_command="mirflag";

    # preparation
    $working_directory=~s/\/$//;

    for (my $i=0;$i<=$#ifs;$i++){
	my $input_dataset=$working_directory."/".$source.".".$ifs[$i];
	for (my $j=0;$j<2;$j++){
	    my $mirflag_command=$miriad_mirflag_command;
	    $mirflag_command.=" vis=".$input_dataset;
	    if ($j==0){
		$mirflag_command.=" stokes=xx,xx,yy";
	    } elsif ($j==1){
		$mirflag_command.=" stokes=yy,xx,yy";
	    }
	    $mirflag_command.=" options=amp,medsed,short";
	    if ($output_debug>0){
		print "executing: $mirflag_command\n";
	    }
	    system $mirflag_command;
	}
    }
}

sub calibrate {
    my ($source,$source_type,$reference_ant,$working_directory,$nifs,@ifs_stuff)=@_;
    my @ifs;
    my @bandifs;
    for (my $i=0;$i<=$nifs;$i++){
	push @bandifs,shift @ifs_stuff;
    }
    @ifs=@ifs_stuff;

    # some MIRIAD commands
    my $miriad_mfcal_command="mfcal";
    my $miriad_gpcal_command="gpcal";

    # preparation
    $working_directory=~s/\/$//;
    my $returncode=0;

    for (my $i=0;$i<=$#ifs;$i++){
	my $calibrated=0;
	my ($do_mfcal,$do_gpcal)=(0,0);
	if ($bandifs[$i]==1){
	    $do_mfcal=1;
	    $do_gpcal=1;
	} else {
	    $do_gpcal=1;
	}
	my $no_qusolve=0;
	my $count_cals=0;
	while($calibrated==0){
	    $count_cals++;
	    if ($count_cals>10){
		# we've tried 10 times, calibration isn't working
		$returncode=-1;
		last;
	    }
	    my $input_dataset=$working_directory."/".$source.".".$ifs[$i];
	    &delete_calibration($input_dataset,$bandifs[$i]);
	    if ($do_mfcal==1){
		my $mfcal_command=$miriad_mfcal_command;
		$mfcal_command.=" vis=".$input_dataset;
		$mfcal_command.=" interval=0.1";
		$mfcal_command.=" refant=".$reference_ant;
		if ($bandifs[$i]==0){
		    $mfcal_command.=" options=nopassol";
		}
		if ($output_debug>0){
		    print "executing: $mfcal_command\n";
		}
		open(MFCAL,"-|")|| exec $mfcal_command." 2>&1";
		my $mfcal_success=1;
		while(<MFCAL>){
		    chomp;
		    my $line=$_;
		    if ($output_debug>1){
			print "MIRIAD: $line\n";
		    }
		    if ($line=~/^\#\#\#\sFatal\sError\:/){
			$mfcal_success=0;
		    }
		}
		close(MFCAL);
		if ($mfcal_success==0){
		    $returncode=-1;
		    last;
		}
	    }
	    $calibrated=1;
	    if ($do_gpcal==1){
		my $gpcal_command=$miriad_gpcal_command;
		$gpcal_command.=" vis=".$input_dataset;
		$gpcal_command.=" interval=0.1";
		$gpcal_command.=" refant=".$reference_ant;
		$gpcal_command.=" options=xyvary";
		if (($source_type ne "f")&&($no_qusolve==0)){
		    $gpcal_command.=",qusolve";
		}
		if ($output_debug>0){
		    print "executing: $gpcal_command\n";
		}
		open(GPCAL,"-|")||exec $gpcal_command." 2>&1";
		while(<GPCAL>){
		    chomp;
		    my $line=$_;
		    if ($output_debug>1){
			print "MIRIAD: $line\n";
		    }
		    if ($line=~/\#\#\#\sWarning/){
			if ($line=~/Failed\sto\sconverge/){
			    $calibrated=0;
			    $do_gpcal=0;
			}
		    } elsif ($line=~/^\s+Ant\s.*nan.*/){
			$calibrated=0;
			$do_gpcal=0;
		    } elsif ($line=~/^\#\#\#\sFatal\sError\:/){
			if ($line=~/Solution\sfor\srequested\sparameters\sis\sdegenerate/){
			    $calibrated=0;
			    $no_qusolve=1;
			    $do_gpcal=1;
			} else {
			    $calibrated=0;
			    $do_gpcal=0;
			}
		    }
		}
		close(GPCAL);
	    }
	}
    }

    return $returncode;
}

sub delete_calibration {
    my ($dataset,$delbandpass)=@_;

    # preparation
    $dataset=~s/\/$//;

    my $remove_command="rm -f ".$dataset;
    my @delete_elements=("gains","leakage");
    if ($delbandpass==1){
	push @delete_elements,"bandpass";
    }
    for (my $i=0;$i<=$#delete_elements;$i++){
	my $delete_command=$remove_command."/".$delete_elements[$i];
	if ($output_debug>0){
	    print "executing: $delete_command\n";
	}
	system $delete_command;
    }
    # get rid of any small gain table in the header
    $remove_command="delhd in=".$dataset."/gains";
    if ($output_debug>0){
	print "executing: $remove_command\n";
    }
    system $remove_command;
}

sub array_to_text {
    # turns an array into a postgresql compliant input
    my (@array)=@_;

    my $string="{";
    # check for letters, but we don't look for 'E' which may be used
    # in exponential numbers
    my $needsquotes=0;
    for (my $i=0;$i<=$#array;$i++){
	if ($array[$i]=~/[a-zA-DF-Z]/){
	    $needsquotes=1;
	}
    }
    for (my $i=0;$i<=$#array;$i++){
	if ($i>0){
	    $string.=",";
	}
	if ($needsquotes==1){
	    $string.="'";
	}
	if (!$array[$i]){
	    if ($needsquotes==1){
		$string.="NA";
	    } else {
		$string.="999";
	    }
	} else {
	    $string.=$array[$i];
	}
	if ($needsquotes==1){
	    $string.="'";
	}
    }
    $string.="}";
    return $string;
}

sub get_parameters {
    # this routine handles the getting of all the nice metadata we want to stick
    # into the database
    my ($source,$rpfits_file,$working_directory,@ifs)=@_;

    # we get what we can from MIRIAD's uvlist
    my $miriad_uvlist_command="uvlist";
    my $rpfits_rpfhdr_command="rpfhdr";

    # the hash we will return to the caller
    my %parameters;

    # preparation
    $working_directory=~s/\/$//;

    for (my $i=0;$i<=$#ifs;$i++){
	my $input_dataset=$working_directory."/".$source.".".$ifs[$i];
	my $logfile="/tmp/uvlist.log";
	# make sure there isn't an old logfile lying around
	my $clear_command="rm -f ".$logfile;
	if ($output_debug>0){
	    print "executing: $clear_command\n";
	}
	system $clear_command;
	my $uvlist_command=$miriad_uvlist_command;
	$uvlist_command.=" vis=".$input_dataset;
	$uvlist_command.=" options=var,full";
	$uvlist_command.=" log=".$logfile;
	if ($output_debug>0){
	    print "executing: $uvlist_command\n";
	}
	system $uvlist_command;
	# check that the logfile has been created
	if (!-e $logfile){
	    if ($output_debug>0){
		print "unable to get output of uvlist!\n";
		exit;
	    }
	}
	open(LOGFILE,$logfile);
	my $started=0;
	my $left_to_get=0;
	my $getting_parameter;
	my @paramsget;
	while(<LOGFILE>){
	    chomp;
	    my $line=$_;
	    # any "(no value set)" will cause us problems, so we fix that
	    $line=~s/\(no\svalue\sset\)/\(novalueset\)/g;
	    if ($line=~/^Header\svariables/){
		$started=1;
	    } elsif ($started==1){
		$line=~s/\:/ /g;
		$line=~s/(\d)(\-)/$1 $2/g;
		my @els=split(/\s+/,$line);
		for (my $j=0;$j<=$#els;$j++){
		    if (($els[$j] eq "airtemp")||($els[$j] eq "antaz")||
			($els[$j] eq "antel")||($els[$j] eq "chi")||
			($els[$j] eq "evector")||($els[$j] eq "jyperk")||
			($els[$j] eq "pntdec")||($els[$j] eq "pntra")||
			($els[$j] eq "relhumid")||($els[$j] eq "smonrms")||
			($els[$j] eq "wind")||($els[$j] eq "winddir")||
			($els[$j] eq "nants")){
			my $this_parameter=$els[$j]."_if".$ifs[$i];
			$j++;
			my $this_value=$els[$j];
			if ($this_value eq "(novalueset)"){
			    $this_value=-999.999; # magic number alert!
			}
			$parameters{$this_parameter}=$this_value;
			if ($output_debug>1){
			    print "got $this_parameter : $parameters{$this_parameter}\n";
			}
		    } elsif ($left_to_get>0){
			if ($els[$j] eq $getting_parameter){
			    next;
			}
			if ($els[$j] eq "(novalueset)"){
			    $left_to_get=0;
			    next;
			}
			push @paramsget,$els[$j];
			$left_to_get--;
			if ($left_to_get==0){
			    $getting_parameter.="_if".$ifs[$i];
			    $parameters{$getting_parameter}=[@paramsget];
			}
		    } elsif (($els[$j] eq "xtsys")||
			     ($els[$j] eq "xyphase")||($els[$j] eq "ytsys")||
			     ($els[$j] eq "xcaljy")||($els[$j] eq "ycaljy")||
			     ($els[$j] eq "xgtp")||($els[$j] eq "ygtp")||
			     ($els[$j] eq "xsdo")||($els[$j] eq "ysdo")){
			if ($left_to_get==0){
			    $getting_parameter=$els[$j];
			    $left_to_get=$parameters{"nants_if$ifs[$i]"};
			    if ($output_debug>1){
				print "starting parameter get for $getting_parameter, $left_to_get values to get\n";
			    }
			    undef @paramsget;
			}
		    }
		}
	    }
	}
	close(LOGFILE);
    }
    
    # we get more parameters from the RPFITS header
    my $rpfhdr_command=$rpfits_rpfhdr_command." ".$rpfits_file;
    if ($output_debug>0){
	print "executing: $rpfhdr_command\n";
    }
    my @attenmm_a;
    my @attenmm_b;
    my @subrefl;
    my @pointcor_az;
    my @pointcor_el;
    open(RPFHDR,"-|")||exec $rpfhdr_command;
    while(<RPFHDR>){
	chomp;
	my $line=$_;
	if ($line=~/^ATTEN/){
	    # mm attenuators
	    $line=~/^ATTEN\s+CA0(.)\sMM\=(\d*)\s(\d*)/;
	    my $antenna=$1;
	    my $mmatten_a=$2;
	    my $mmatten_b=$3;
	    $attenmm_a[$antenna-1]=$mmatten_a;
	    $attenmm_b[$antenna-1]=$mmatten_b;
	} elsif ($line=~/^SUBREFL/){
	    # subreflector position
	    $line=~/^SUBREFL\s+CA0(.)\sPOS\=\s*(.*)\sTILT.*/;
	    my $antenna=$1;
	    my $position=$2;
	    $subrefl[$antenna-1]=$position;
	} elsif ($line=~/^POINTCOR/){
	    # pointing correction
	    $line=~/^POINTCOR\s+CA0(.)\sAz\=\s*(.*)\sEl\=\s*(.*)\s.*\/.*/;
	    my $antenna=$1;
	    my $azcor=$2;
	    my $elcor=$3;
	    $pointcor_az[$antenna-1]=$azcor;
	    $pointcor_el[$antenna-1]=$elcor;
	} elsif ($line=~/^CORR_CFG/){
	    # correlator configuration
	    $line=~/^CORR_CFG\=\s*(.*)$/;
	    $parameters{'corr_cfg'}=$1;
	} elsif ($line=~/^POINTINF/){
	    # last pointing information
	    $line=~/^POINTINF\s*(.*)\sSOURCE\=\s*(.*)\s*AZ\=\s(.*)\sEL\=\s(.*)/;
	    $parameters{'point_time'}=$1;
	    $parameters{'point_source'}=$2;
	    $parameters{'point_az'}=$3;
	    $parameters{'point_el'}=$4;
	    $parameters{'point_source'}=~s/\s*$//;
	}
    }
    close(RPFHDR);
    $parameters{'attenmm_a'}=[@attenmm_a];
    $parameters{'attenmm_b'}=[@attenmm_b];
    $parameters{'subrefl'}=[@subrefl];
    # do we need to add the pointing arrays
    if ($parameters{'point_time'}){
	$parameters{'pointcor_az'}=[@pointcor_az];
	$parameters{'pointcor_el'}=[@pointcor_el];
    }
    
    return (\%parameters);
}

sub minmax {
    # get the minimum and maximum values in an array
    my @array=@_;

    my $minval=$array[0];
    my $maxval=$array[0];
    for (my $i=0;$i<=$#array;$i++){
	if ($array[$i]<$minval){
	    $minval=$array[$i];
	}
	if ($array[$i]>$maxval){
	    $maxval=$array[$i];
	}
    }
    return ($minval,$maxval);
}

sub observation_set_unusable {
    my ($dbh,$id)=@_;

#    if ($redo_bandpass==1){
#	return;
#    }
    
#    my $set_query="UPDATE observation SET num_processed=-1 WHERE id=$id";
    my $set_query="UPDATE observation SET usable=false WHERE id=$id";
    if ($output_debug>0){
	print "query: $set_query\n";
    }
    my $set=$dbh->prepare($set_query);
    $set->execute;
    $set->finish;
}

sub fits_to_miriad {
    my ($rpfits_chunk,$source,$work_directory,@ifs)=@_;

    # some commands
    my $miriad_atlod_command="atlod";
    my $miriad_uvsplit_command="uvsplit";

    # ATLOD into MIRIAD format
    my @output_files;
    $work_directory=~s/\/$//;
    $rpfits_chunk=~s/^.*\/(.*)$/$1/;
    my $current_directory=getcwd();
    chdir $work_directory;
    my $input_file=$rpfits_chunk;
    my $num_success=0;
    for (my $i=0;$i<=$#ifs;$i++){
	my $this_if=$ifs[$i]+1;
	my $output_file=$input_file."_if".$this_if.".uv";
	my $miriad_options="birdie,rfiflag,noauto,nocacal,xycorr,opcorr";
	my $atlod_command=$miriad_atlod_command;
	$atlod_command.=" in=".$input_file;
	$atlod_command.=" out=".$output_file;
	$atlod_command.=" options=".$miriad_options;
	$atlod_command.=" ifsel=".$this_if;
	if ($output_debug>0){
	    print "executing command: $atlod_command\n";
	}
	my $rc=&run_miriad_command($atlod_command);
	if ($rc==0){
	    my $uvsplit_command=$miriad_uvsplit_command;
	    $uvsplit_command.=" vis=".$output_file;
	    $uvsplit_command.=" options=clobber";
	    if ($output_debug>0){
		print "executing command: $uvsplit_command\n";
	    }
	    $rc=&run_miriad_command($uvsplit_command);
	    my $remove_command="rm -rf ".$output_file;
	    if ($output_debug>0){
		print "executing command: $remove_command\n";
	    }
	    system $remove_command;
	}
	if ($rc==0){
	    $num_success++;
	}
    }

    chdir $current_directory;

    if ($num_success==0){
	# something went wrong
	return -1;
    }
}

sub run_miriad_command {
    my ($miriad_command)=@_;
    
    my $rc=0;
    open(MIRIAD_COMMAND,"-|")||exec $miriad_command." 2>&1";
    while(<MIRIAD_COMMAND>){
	chomp;
	my $line=$_;
	if ($output_debug>1){
	    print "MIRIAD: $line\n";
	}
	if ($line=~/^\#\#\#\sFatal\sError/){
	    $rc=-1;
	}
    }
    close(MIRIAD_COMMAND);

    return $rc;
}

sub postgresql_array_get {
    my ($arraystring)=@_;
    # turns a postgresql array string into a perl array
    if ($output_debug>2){
	print "attempting to turn $arraystring into array\n";
    }
    $arraystring=~s/[\{\}]//g;
    my @els=split(/\,/,$arraystring);
    return @els;
}

sub extract_rpfits {
    my ($rpfits_file,$headernum,$source,$datetime,$work_directory)=@_;

    # some commands
    my $rpfits_extract_command="/n/ste616/bin/rpfex";

    # try to find the RPFITS file
    my @search_paths=("/DATA/ARCHIVE_1/ATCA/archive");
    my $found_file="";
    $rpfits_file=~s/^.*\/(.*)$/$1/;

    for (my $i=0;$i<=$#search_paths;$i++){
	$search_paths[$i]=~s/\/$//;
	my $search_file=$search_paths[$i]."/".$rpfits_file;
	if (-e $search_file){
	    $found_file=$search_file;
	    last;
	}
    }
    if ($found_file eq ""){
	# couldn't find this file!
	print "unable to find RPFITS file $rpfits_file from local repository!\n";

	# try to get this file from the ATOA
	$rpfits_file=~/(\d{4})\-(\d{2})\-(\d{2})\_(.*)/;
	my $rpfits_year=$1;
	my $rpfits_month=$2;
	my @rpfits_archive_search=("/data/ATOA_1/ATCA",
				   "/data/ATOA_2/ATCA");
	for (my $i=0;$i<=$#rpfits_archive_search;$i++){
#	    my $rpfits_archive_path="/data/ATOA_1/ATCA";
	    my $rpfits_archive_path=$rpfits_archive_search[$i];
#	    if (($rpfits_year>=2009)&&($rpfits_month>=6)){
#		$rpfits_archive_path="/data/ATOA_2/ATCA";
#	    }
	    $rpfits_archive_path.="/".$rpfits_year."-".$rpfits_month;

	    my $copy_command="rsync -av ste616\@horus:".$rpfits_archive_path."/";
	    $copy_command.=$rpfits_file;
	    $copy_command.=" ".$work_directory."/";
	    if ($output_debug>0){
		print "executing: $copy_command\n";
	    }
	    system $copy_command;
	    
	    $found_file=$work_directory."/".$rpfits_file;
	    if (!-e $found_file){
		$found_file="";
	    } else {
		last;
	    }
	}
    }

    if ($found_file eq ""){
	# still couldn't find the file
	print "unable to find RPFITS file $rpfits_file from any repository!\n";
	
	return "";
    }

    # extract out the data
    $datetime=~s/\s/_/g;
    my $output_file=$source."_".$datetime.".fits";
    $work_directory=~s/\/$//;
    my $output_full=$work_directory."/".$output_file;
    my $extract_command=$rpfits_extract_command;
    $extract_command.=" -x".$headernum.":".$headernum;
    $extract_command.=" -o ".$output_full;
    $extract_command.=" ".$found_file;

    if ($output_debug>0){
	print "extracting source RPFITS headers with command:\n";
	print $extract_command."\n";
    }
    if (!-e $output_full){
	system $extract_command;
    } # otherwise the file already exists and has already been extracted

    # return the location of the file we just extracted
    return $output_full;
    
}

sub is_planet {
    my ($source)=@_;

    $source=~tr/A-Z/a-z/; # change to lowercase

    my $planet=0;
    my @planets=qw/mercury venus earth mars jupiter saturn uranus neptune/;

    for (my $i=0;$i<=$#planets;$i++){
	if ($source eq $planets[$i]){
	    $planet=1;
	    last;
	}
    }

    return $planet;
}

sub get_id_observation_calibrators_info {
    my ($dbh,$get_id)=@_;

    my $get_query="SELECT * FROM observation,calibrators WHERE id=".$get_id;
    $get_query.=" AND calibrators.name=observation.source";
    my $get=$dbh->prepare($get_query);
    $get->execute;
    my @get_response;
    while (@get_response=$get->fetchrow){
	last;
    }
    $get->finish;

    return @get_response;
}

sub which_wavelength {
    my ($frequency)=@_;

    my $wavelength;
    if (($frequency>=$highlow_21cm[0])&&
	($frequency<$highlow_21cm[1])){
	$wavelength="21cm";
    } elsif (($frequency>=$highlow_13cm[0])&&
	     ($frequency<$highlow_13cm[1])){
	$wavelength="13cm";
    } elsif (($frequency>=$highlow_6cm[0])&&
	     ($frequency<$highlow_6cm[1])){
	$wavelength="6cm";
    } elsif (($frequency>=$highlow_3cm[0])&&
	     ($frequency<$highlow_3cm[1])){
	$wavelength="3cm";
    } elsif (($frequency>=$highlow_12mm[0])&&
	     ($frequency<$highlow_12mm[1])){
	$wavelength="12mm";
    } elsif (($frequency>=$highlow_7mm[0])&&
	     ($frequency<$highlow_7mm[1])){
	$wavelength="7mm";
    } elsif (($frequency>=$highlow_3mm[0])&&
	     ($frequency<$highlow_3mm[1])){
	$wavelength="3mm";
    }

    return $wavelength;
}

sub shrink_path {
    my $full_path=$_[0];

    if ($full_path=~/^\.\.\//){
	# this path starts with a relative directory,
	# so has probably already been shrunk, so we simply
	# return straight away
	return $full_path;
    }

    my $currdir=getcwd;
    my @eldir=split(/\/+/,$currdir);
    for (my $i=1;$i<=$#eldir;$i++){
	if ($full_path=~/^\/$eldir[$i].*/){
	    $full_path=~s/^\/$eldir[$i]//;
	} else {
	    $full_path="..".$full_path;
	}
    }
    
    if (($full_path!~/^\/.*\/.*/)&&
	($full_path!~/^\.\.\/.*/)){
	$full_path=".".$full_path;
    }

    return $full_path;
}

sub has_phase_calibrated_dataset {
    my ($dbh,@processed_ids)=@_;

    if ($#processed_ids<0){
	return 0;
    }

    my $sel_query="SELECT id FROM fluxdata WHERE (";
    for (my $i=0;$i<=$#processed_ids;$i++){
	if ($i>0){
	    $sel_query.=" OR ";
	}
	$sel_query.="id=".$processed_ids[$i];
    }
    $sel_query.=") AND processed_as_phase=TRUE";
    if ($output_debug>0){
	print "executing query: $sel_query\n";
    }
    my $sel=$dbh->prepare($sel_query);
    $sel->execute;

    my $nproc=0;
    while (my @sel_response=$sel->fetchrow){
	$nproc++;
    }
    $sel->finish;

    return $nproc;
}

sub get_database_datasets {
    my ($dbh,$working_directory,@ids)=@_;
    
    # preparation
    $working_directory=~s/\/$//;
    my @frequencies_found;

    my $database_server_login="postgres\@localhost";
    my $database_server_port=2222;
    my $database_server_path="/var/lib/postgresql";

    # extract the flux calibrator dataset and find out what frequencies
    # they have
    for (my $i=0;$i<=$#ids;$i++){
	my $dataset_name="dataset_".$ids[$i].".tar.bz2";
	my $extracted_dataset=$database_server_path."/".$dataset_name;
	my $gti_query="SELECT fluxdata.frequency,lo_export(fluxdata.uv_data,'".
	    $extracted_dataset."') FROM fluxdata WHERE fluxdata.id=".
	    $ids[$i];
	if ($output_debug>1){
	    print "executing query: $gti_query\n";
	}
	my $gti=$dbh->prepare($gti_query);
	$gti->execute;

	while (my @gti_response=$gti->fetchrow){
	    my $thisfreq=$gti_response[0];
	    my $found=0;
	    for (my $j=0;$j<=$#frequencies_found;$j++){
		if ($frequencies_found[$j]==$thisfreq){
		    $found=1;
		    last;
		}
	    }
	    if ($found==0){
		push @frequencies_found,$thisfreq;
	    }
	}

	# get the data back here
	my $scp_command="scp -P ".$database_server_port;
	$scp_command.=" ".$database_server_login.":".$extracted_dataset;
	$scp_command.=" ".$working_directory."/.";
	if ($output_debug>1){
	    print "executing: $scp_command\n";
	}
	system $scp_command;
	# and remove the remote dataset
	my $srm_command="ssh -p ".$database_server_port;
	$srm_command.=" ".$database_server_login;
	$srm_command.=" rm -rf ".$extracted_dataset;
	if ($output_debug>1){
	    print "executing: $srm_command\n";
	}
	system $srm_command;

	# unarchive the data
	my $input_archive=$working_directory."/".$dataset_name;
	my $untar_command="tar -C ".$working_directory;
	$untar_command.=" -xjf ".$input_archive;
	if ($output_debug>1){
	    print "executing: $untar_command\n";
	}
	system $untar_command;
	# remove the archive
	my $lrm_command="rm -f ".$input_archive;
	if ($output_debug>1){
	    print "executing: $lrm_command\n";
	}
	if ($delete_process==1){
	    system $lrm_command;
	}
    }

    return @frequencies_found;
}
